2024-11-11 07:22:03,142 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 07:22:03,157 main DEBUG Took 0.012937 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 07:22:03,157 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 07:22:03,158 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 07:22:03,158 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 07:22:03,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,167 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 07:22:03,178 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,180 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,181 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,182 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,183 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,183 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,184 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,184 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,184 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,185 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,185 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,185 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,186 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,187 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,187 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:22:03,188 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,188 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 07:22:03,189 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:22:03,191 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 07:22:03,192 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 07:22:03,193 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 07:22:03,194 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 07:22:03,194 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 07:22:03,202 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 07:22:03,204 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 07:22:03,205 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 07:22:03,206 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 07:22:03,206 main DEBUG createAppenders(={Console}) 2024-11-11 07:22:03,207 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 07:22:03,207 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 07:22:03,207 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 07:22:03,208 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 07:22:03,208 main DEBUG OutputStream closed 2024-11-11 07:22:03,208 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 07:22:03,209 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 07:22:03,209 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 07:22:03,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 07:22:03,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 07:22:03,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 07:22:03,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 07:22:03,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 07:22:03,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 07:22:03,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 07:22:03,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 07:22:03,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 07:22:03,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 07:22:03,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 07:22:03,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 07:22:03,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 07:22:03,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 07:22:03,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 07:22:03,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 07:22:03,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 07:22:03,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 07:22:03,306 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 07:22:03,306 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 07:22:03,306 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 07:22:03,307 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T07:22:03,575 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77 2024-11-11 07:22:03,578 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 07:22:03,578 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T07:22:03,588 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-11T07:22:03,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T07:22:03,622 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143, deleteOnExit=true 2024-11-11T07:22:03,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T07:22:03,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/test.cache.data in system properties and HBase conf 2024-11-11T07:22:03,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T07:22:03,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/hadoop.log.dir in system properties and HBase conf 2024-11-11T07:22:03,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T07:22:03,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T07:22:03,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T07:22:03,721 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T07:22:03,823 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T07:22:03,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T07:22:03,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T07:22:03,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T07:22:03,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T07:22:03,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T07:22:03,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T07:22:03,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T07:22:03,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T07:22:03,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T07:22:03,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/nfs.dump.dir in system properties and HBase conf 2024-11-11T07:22:03,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/java.io.tmpdir in system properties and HBase conf 2024-11-11T07:22:03,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T07:22:03,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T07:22:03,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T07:22:04,699 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T07:22:04,786 INFO [Time-limited test {}] log.Log(170): Logging initialized @2479ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T07:22:04,878 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T07:22:04,941 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T07:22:04,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T07:22:04,966 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T07:22:04,967 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T07:22:04,984 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T07:22:04,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/hadoop.log.dir/,AVAILABLE} 2024-11-11T07:22:04,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T07:22:05,188 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/java.io.tmpdir/jetty-localhost-38489-hadoop-hdfs-3_4_1-tests_jar-_-any-4087277458507200407/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T07:22:05,204 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38489} 2024-11-11T07:22:05,204 INFO [Time-limited test {}] server.Server(415): Started @2898ms 2024-11-11T07:22:05,643 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T07:22:05,651 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T07:22:05,652 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T07:22:05,652 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T07:22:05,652 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T07:22:05,653 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/hadoop.log.dir/,AVAILABLE} 2024-11-11T07:22:05,654 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T07:22:05,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/java.io.tmpdir/jetty-localhost-33729-hadoop-hdfs-3_4_1-tests_jar-_-any-2127722516741226606/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T07:22:05,784 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:33729} 2024-11-11T07:22:05,784 INFO [Time-limited test {}] server.Server(415): Started @3478ms 2024-11-11T07:22:05,843 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T07:22:06,363 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143/data/data1/current/BP-743239414-172.17.0.2-1731309724451/current, will proceed with Du for space computation calculation, 2024-11-11T07:22:06,363 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143/data/data2/current/BP-743239414-172.17.0.2-1731309724451/current, will proceed with Du for space computation calculation, 2024-11-11T07:22:06,404 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T07:22:06,467 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x73afc70fcbd35af3 with lease ID 0x376888826e141560: Processing first storage report for DS-74a6e686-2d71-4357-ae40-76976cd3e84c from datanode DatanodeRegistration(127.0.0.1:40127, datanodeUuid=0ee03f57-8736-47de-a7ba-6f3841aa7eb6, infoPort=34931, infoSecurePort=0, ipcPort=42245, storageInfo=lv=-57;cid=testClusterID;nsid=1851262847;c=1731309724451) 2024-11-11T07:22:06,469 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x73afc70fcbd35af3 with lease ID 0x376888826e141560: from storage DS-74a6e686-2d71-4357-ae40-76976cd3e84c node DatanodeRegistration(127.0.0.1:40127, datanodeUuid=0ee03f57-8736-47de-a7ba-6f3841aa7eb6, infoPort=34931, infoSecurePort=0, ipcPort=42245, storageInfo=lv=-57;cid=testClusterID;nsid=1851262847;c=1731309724451), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-11T07:22:06,470 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x73afc70fcbd35af3 with lease ID 0x376888826e141560: Processing first storage report for DS-035b3570-4721-4a55-a397-f9506062efa9 from datanode DatanodeRegistration(127.0.0.1:40127, datanodeUuid=0ee03f57-8736-47de-a7ba-6f3841aa7eb6, infoPort=34931, infoSecurePort=0, ipcPort=42245, storageInfo=lv=-57;cid=testClusterID;nsid=1851262847;c=1731309724451) 2024-11-11T07:22:06,470 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x73afc70fcbd35af3 with lease ID 0x376888826e141560: from storage DS-035b3570-4721-4a55-a397-f9506062efa9 node DatanodeRegistration(127.0.0.1:40127, datanodeUuid=0ee03f57-8736-47de-a7ba-6f3841aa7eb6, infoPort=34931, infoSecurePort=0, ipcPort=42245, storageInfo=lv=-57;cid=testClusterID;nsid=1851262847;c=1731309724451), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T07:22:06,532 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77 2024-11-11T07:22:06,626 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143/zookeeper_0, clientPort=53879, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T07:22:06,637 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53879 2024-11-11T07:22:06,648 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:22:06,653 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:22:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741825_1001 (size=7) 2024-11-11T07:22:07,304 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 with version=8 2024-11-11T07:22:07,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/hbase-staging 2024-11-11T07:22:07,409 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T07:22:07,681 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f334e8b0fbdf:0 server-side Connection retries=45 2024-11-11T07:22:07,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:22:07,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T07:22:07,696 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T07:22:07,696 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:22:07,697 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T07:22:07,832 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T07:22:07,894 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T07:22:07,903 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T07:22:07,907 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T07:22:07,935 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3832 (auto-detected) 2024-11-11T07:22:07,936 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T07:22:07,956 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39865 2024-11-11T07:22:07,979 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39865 connecting to ZooKeeper ensemble=127.0.0.1:53879 2024-11-11T07:22:08,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398650x0, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T07:22:08,016 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39865-0x10105e7d61c0000 connected 2024-11-11T07:22:08,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:22:08,048 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:22:08,059 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:22:08,063 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537, hbase.cluster.distributed=false 2024-11-11T07:22:08,085 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T07:22:08,090 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39865 2024-11-11T07:22:08,090 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39865 2024-11-11T07:22:08,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39865 2024-11-11T07:22:08,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39865 2024-11-11T07:22:08,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39865 2024-11-11T07:22:08,230 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f334e8b0fbdf:0 server-side Connection retries=45 2024-11-11T07:22:08,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:22:08,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T07:22:08,232 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T07:22:08,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:22:08,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T07:22:08,235 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T07:22:08,238 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T07:22:08,239 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39329 2024-11-11T07:22:08,242 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39329 connecting to ZooKeeper ensemble=127.0.0.1:53879 2024-11-11T07:22:08,244 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:22:08,248 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:22:08,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:393290x0, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T07:22:08,259 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:393290x0, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:22:08,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39329-0x10105e7d61c0001 connected 2024-11-11T07:22:08,264 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T07:22:08,274 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T07:22:08,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T07:22:08,282 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T07:22:08,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39329 2024-11-11T07:22:08,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39329 2024-11-11T07:22:08,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39329 2024-11-11T07:22:08,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39329 2024-11-11T07:22:08,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39329 2024-11-11T07:22:08,307 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f334e8b0fbdf:39865 2024-11-11T07:22:08,308 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:08,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:22:08,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:22:08,317 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:08,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T07:22:08,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:08,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:08,340 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T07:22:08,341 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f334e8b0fbdf,39865,1731309727465 from backup master directory 2024-11-11T07:22:08,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:08,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:22:08,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:22:08,347 WARN [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T07:22:08,347 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:08,349 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T07:22:08,350 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T07:22:08,407 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/hbase.id] with ID: 81636d0b-cdc2-4858-a30b-4ad347eabbaf 2024-11-11T07:22:08,407 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/.tmp/hbase.id 2024-11-11T07:22:08,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741826_1002 (size=42) 2024-11-11T07:22:08,819 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/.tmp/hbase.id]:[hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/hbase.id] 2024-11-11T07:22:08,862 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:22:08,866 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T07:22:08,890 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-11T07:22:08,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:08,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:08,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741827_1003 (size=196) 2024-11-11T07:22:09,331 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:22:09,334 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T07:22:09,352 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:09,356 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T07:22:09,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741828_1004 (size=1189) 2024-11-11T07:22:09,811 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store 2024-11-11T07:22:09,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741829_1005 (size=34) 2024-11-11T07:22:10,236 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T07:22:10,240 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:10,242 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T07:22:10,242 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:22:10,242 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:22:10,244 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T07:22:10,245 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:22:10,245 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:22:10,247 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731309730242Disabling compacts and flushes for region at 1731309730242Disabling writes for close at 1731309730245 (+3 ms)Writing region close event to WAL at 1731309730245Closed at 1731309730245 2024-11-11T07:22:10,250 WARN [master/f334e8b0fbdf:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/.initializing 2024-11-11T07:22:10,250 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/WALs/f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:10,260 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T07:22:10,276 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f334e8b0fbdf%2C39865%2C1731309727465, suffix=, logDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/WALs/f334e8b0fbdf,39865,1731309727465, archiveDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/oldWALs, maxLogs=10 2024-11-11T07:22:10,300 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/WALs/f334e8b0fbdf,39865,1731309727465/f334e8b0fbdf%2C39865%2C1731309727465.1731309730280, exclude list is [], retry=0 2024-11-11T07:22:10,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-74a6e686-2d71-4357-ae40-76976cd3e84c,DISK] 2024-11-11T07:22:10,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T07:22:10,358 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/WALs/f334e8b0fbdf,39865,1731309727465/f334e8b0fbdf%2C39865%2C1731309727465.1731309730280 2024-11-11T07:22:10,359 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34931:34931)] 2024-11-11T07:22:10,359 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:22:10,360 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:10,363 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,364 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T07:22:10,434 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:10,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:10,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T07:22:10,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:10,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:10,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T07:22:10,444 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:10,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:10,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T07:22:10,448 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:10,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:10,450 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,453 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,454 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,460 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,461 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,465 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T07:22:10,469 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:22:10,474 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:22:10,475 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75291907, jitterRate=0.1219368427991867}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T07:22:10,480 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731309730376Initializing all the Stores at 1731309730379 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731309730379Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309730380 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309730380Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309730380Cleaning up temporary data from old regions at 1731309730461 (+81 ms)Region opened successfully at 1731309730480 (+19 ms) 2024-11-11T07:22:10,481 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T07:22:10,518 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc20332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f334e8b0fbdf/172.17.0.2:0 2024-11-11T07:22:10,551 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T07:22:10,564 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T07:22:10,564 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T07:22:10,568 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T07:22:10,569 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T07:22:10,575 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T07:22:10,575 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T07:22:10,605 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T07:22:10,614 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T07:22:10,616 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T07:22:10,619 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T07:22:10,621 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T07:22:10,622 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T07:22:10,625 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T07:22:10,628 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T07:22:10,630 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T07:22:10,631 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T07:22:10,633 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T07:22:10,652 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T07:22:10,654 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T07:22:10,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T07:22:10,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T07:22:10,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:10,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:10,661 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f334e8b0fbdf,39865,1731309727465, sessionid=0x10105e7d61c0000, setting cluster-up flag (Was=false) 2024-11-11T07:22:10,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:10,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:10,681 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T07:22:10,683 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:10,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:10,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:10,697 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T07:22:10,699 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:10,707 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T07:22:10,788 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T07:22:10,798 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(746): ClusterId : 81636d0b-cdc2-4858-a30b-4ad347eabbaf 2024-11-11T07:22:10,799 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T07:22:10,801 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T07:22:10,805 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T07:22:10,807 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T07:22:10,807 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T07:22:10,810 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T07:22:10,811 DEBUG [RS:0;f334e8b0fbdf:39329 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478a51f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f334e8b0fbdf/172.17.0.2:0 2024-11-11T07:22:10,811 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f334e8b0fbdf,39865,1731309727465 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T07:22:10,819 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f334e8b0fbdf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:22:10,819 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f334e8b0fbdf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:22:10,820 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f334e8b0fbdf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:22:10,820 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f334e8b0fbdf:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:22:10,820 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f334e8b0fbdf:0, corePoolSize=10, maxPoolSize=10 2024-11-11T07:22:10,820 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,820 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f334e8b0fbdf:0, corePoolSize=2, maxPoolSize=2 2024-11-11T07:22:10,821 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,822 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731309760822 2024-11-11T07:22:10,824 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T07:22:10,825 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T07:22:10,826 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T07:22:10,826 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T07:22:10,827 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f334e8b0fbdf:39329 2024-11-11T07:22:10,830 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T07:22:10,830 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T07:22:10,831 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T07:22:10,831 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T07:22:10,831 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T07:22:10,831 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T07:22:10,831 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T07:22:10,833 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:10,833 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T07:22:10,834 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(2659): reportForDuty to master=f334e8b0fbdf,39865,1731309727465 with port=39329, startcode=1731309728191 2024-11-11T07:22:10,832 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,835 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T07:22:10,837 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T07:22:10,837 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T07:22:10,842 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T07:22:10,842 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T07:22:10,849 DEBUG [RS:0;f334e8b0fbdf:39329 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T07:22:10,856 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f334e8b0fbdf:0:becomeActiveMaster-HFileCleaner.large.0-1731309730844,5,FailOnTimeoutGroup] 2024-11-11T07:22:10,857 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f334e8b0fbdf:0:becomeActiveMaster-HFileCleaner.small.0-1731309730857,5,FailOnTimeoutGroup] 2024-11-11T07:22:10,857 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,858 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T07:22:10,859 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,860 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741831_1007 (size=1321) 2024-11-11T07:22:10,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41303, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T07:22:10,894 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39865 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:10,897 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39865 {}] master.ServerManager(517): Registering regionserver=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:10,915 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:22:10,915 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40463 2024-11-11T07:22:10,915 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T07:22:10,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T07:22:10,920 DEBUG [RS:0;f334e8b0fbdf:39329 {}] zookeeper.ZKUtil(111): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:10,920 WARN [RS:0;f334e8b0fbdf:39329 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T07:22:10,921 INFO [RS:0;f334e8b0fbdf:39329 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T07:22:10,921 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/WALs/f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:10,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f334e8b0fbdf,39329,1731309728191] 2024-11-11T07:22:10,949 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T07:22:10,962 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T07:22:10,968 INFO [RS:0;f334e8b0fbdf:39329 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T07:22:10,968 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,969 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T07:22:10,975 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T07:22:10,976 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,976 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,977 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,977 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,977 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,977 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,977 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f334e8b0fbdf:0, corePoolSize=2, maxPoolSize=2 2024-11-11T07:22:10,977 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,978 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,978 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,978 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,978 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,978 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f334e8b0fbdf:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:22:10,978 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f334e8b0fbdf:0, corePoolSize=3, maxPoolSize=3 2024-11-11T07:22:10,978 DEBUG [RS:0;f334e8b0fbdf:39329 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0, corePoolSize=3, maxPoolSize=3 2024-11-11T07:22:10,979 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,979 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,980 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,980 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,980 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:10,980 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39329,1731309728191-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T07:22:11,001 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T07:22:11,003 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39329,1731309728191-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:11,003 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:11,003 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.Replication(171): f334e8b0fbdf,39329,1731309728191 started 2024-11-11T07:22:11,024 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:11,024 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1482): Serving as f334e8b0fbdf,39329,1731309728191, RpcServer on f334e8b0fbdf/172.17.0.2:39329, sessionid=0x10105e7d61c0001 2024-11-11T07:22:11,025 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T07:22:11,026 DEBUG [RS:0;f334e8b0fbdf:39329 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:11,026 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f334e8b0fbdf,39329,1731309728191' 2024-11-11T07:22:11,026 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T07:22:11,027 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T07:22:11,028 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T07:22:11,028 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T07:22:11,028 DEBUG [RS:0;f334e8b0fbdf:39329 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:11,028 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f334e8b0fbdf,39329,1731309728191' 2024-11-11T07:22:11,028 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T07:22:11,029 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T07:22:11,030 DEBUG [RS:0;f334e8b0fbdf:39329 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T07:22:11,030 INFO [RS:0;f334e8b0fbdf:39329 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T07:22:11,030 INFO [RS:0;f334e8b0fbdf:39329 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T07:22:11,135 INFO [RS:0;f334e8b0fbdf:39329 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T07:22:11,139 INFO [RS:0;f334e8b0fbdf:39329 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f334e8b0fbdf%2C39329%2C1731309728191, suffix=, logDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/WALs/f334e8b0fbdf,39329,1731309728191, archiveDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/oldWALs, maxLogs=32 2024-11-11T07:22:11,156 DEBUG [RS:0;f334e8b0fbdf:39329 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/WALs/f334e8b0fbdf,39329,1731309728191/f334e8b0fbdf%2C39329%2C1731309728191.1731309731141, exclude list is [], retry=0 2024-11-11T07:22:11,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-74a6e686-2d71-4357-ae40-76976cd3e84c,DISK] 2024-11-11T07:22:11,166 INFO [RS:0;f334e8b0fbdf:39329 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/WALs/f334e8b0fbdf,39329,1731309728191/f334e8b0fbdf%2C39329%2C1731309728191.1731309731141 2024-11-11T07:22:11,166 DEBUG [RS:0;f334e8b0fbdf:39329 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34931:34931)] 2024-11-11T07:22:11,263 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T07:22:11,264 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:22:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741833_1009 (size=32) 2024-11-11T07:22:11,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:11,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T07:22:11,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T07:22:11,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:11,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:11,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T07:22:11,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T07:22:11,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:11,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:11,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T07:22:11,691 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T07:22:11,691 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:11,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:11,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T07:22:11,695 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T07:22:11,695 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:11,696 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:11,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T07:22:11,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740 2024-11-11T07:22:11,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740 2024-11-11T07:22:11,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T07:22:11,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T07:22:11,703 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:22:11,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T07:22:11,709 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:22:11,709 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70454841, jitterRate=0.0498589426279068}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:22:11,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731309731676Initializing all the Stores at 1731309731678 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731309731678Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731309731679 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309731679Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731309731679Cleaning up temporary data from old regions at 1731309731702 (+23 ms)Region opened successfully at 1731309731712 (+10 ms) 2024-11-11T07:22:11,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T07:22:11,712 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T07:22:11,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T07:22:11,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T07:22:11,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T07:22:11,714 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T07:22:11,714 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731309731712Disabling compacts and flushes for region at 1731309731712Disabling writes for close at 1731309731713 (+1 ms)Writing region close event to WAL at 1731309731713Closed at 1731309731714 (+1 ms) 2024-11-11T07:22:11,717 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T07:22:11,718 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T07:22:11,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T07:22:11,732 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T07:22:11,734 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T07:22:11,887 DEBUG [f334e8b0fbdf:39865 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T07:22:11,899 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:11,905 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f334e8b0fbdf,39329,1731309728191, state=OPENING 2024-11-11T07:22:11,912 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T07:22:11,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:11,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:22:11,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:22:11,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:22:11,916 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T07:22:11,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:22:12,094 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T07:22:12,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42369, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T07:22:12,110 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T07:22:12,110 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T07:22:12,111 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T07:22:12,115 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f334e8b0fbdf%2C39329%2C1731309728191.meta, suffix=.meta, logDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/WALs/f334e8b0fbdf,39329,1731309728191, archiveDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/oldWALs, maxLogs=32 2024-11-11T07:22:12,138 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/WALs/f334e8b0fbdf,39329,1731309728191/f334e8b0fbdf%2C39329%2C1731309728191.meta.1731309732117.meta, exclude list is [], retry=0 2024-11-11T07:22:12,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40127,DS-74a6e686-2d71-4357-ae40-76976cd3e84c,DISK] 2024-11-11T07:22:12,146 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/WALs/f334e8b0fbdf,39329,1731309728191/f334e8b0fbdf%2C39329%2C1731309728191.meta.1731309732117.meta 2024-11-11T07:22:12,146 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34931:34931)] 2024-11-11T07:22:12,147 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:22:12,149 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T07:22:12,153 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T07:22:12,159 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T07:22:12,165 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T07:22:12,165 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:12,166 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T07:22:12,166 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T07:22:12,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T07:22:12,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T07:22:12,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:12,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:12,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T07:22:12,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T07:22:12,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:12,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:12,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T07:22:12,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T07:22:12,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:12,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:12,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T07:22:12,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T07:22:12,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:12,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:22:12,179 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T07:22:12,181 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740 2024-11-11T07:22:12,183 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740 2024-11-11T07:22:12,186 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T07:22:12,186 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T07:22:12,187 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:22:12,189 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T07:22:12,191 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66959093, jitterRate=-0.0022317618131637573}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:22:12,191 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T07:22:12,192 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731309732166Writing region info on filesystem at 1731309732167 (+1 ms)Initializing all the Stores at 1731309732169 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731309732169Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731309732169Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309732169Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731309732169Cleaning up temporary data from old regions at 1731309732186 (+17 ms)Running coprocessor post-open hooks at 1731309732191 (+5 ms)Region opened successfully at 1731309732192 (+1 ms) 2024-11-11T07:22:12,200 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731309732084 2024-11-11T07:22:12,212 DEBUG [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T07:22:12,212 INFO [RS_OPEN_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T07:22:12,215 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:12,217 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f334e8b0fbdf,39329,1731309728191, state=OPEN 2024-11-11T07:22:12,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T07:22:12,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T07:22:12,224 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:22:12,224 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:22:12,225 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:12,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T07:22:12,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f334e8b0fbdf,39329,1731309728191 in 308 msec 2024-11-11T07:22:12,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T07:22:12,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 510 msec 2024-11-11T07:22:12,239 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T07:22:12,240 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T07:22:12,260 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:12,262 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:12,284 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:12,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41331, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:12,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5690 sec 2024-11-11T07:22:12,307 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731309732307, completionTime=-1 2024-11-11T07:22:12,310 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T07:22:12,311 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T07:22:12,342 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-11T07:22:12,342 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731309792342 2024-11-11T07:22:12,342 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731309852342 2024-11-11T07:22:12,343 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-11T07:22:12,345 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39865,1731309727465-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:12,345 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39865,1731309727465-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:12,345 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39865,1731309727465-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:12,346 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f334e8b0fbdf:39865, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:12,347 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:12,347 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:12,353 DEBUG [master/f334e8b0fbdf:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T07:22:12,374 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.026sec 2024-11-11T07:22:12,375 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T07:22:12,376 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T07:22:12,377 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T07:22:12,377 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T07:22:12,378 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T07:22:12,378 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39865,1731309727465-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T07:22:12,379 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39865,1731309727465-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T07:22:12,387 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T07:22:12,388 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T07:22:12,389 INFO [master/f334e8b0fbdf:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f334e8b0fbdf,39865,1731309727465-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:22:12,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:12,409 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T07:22:12,410 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T07:22:12,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:12,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:12,423 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:12,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775efbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:12,434 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:12,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:12,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:12,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:12,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:12,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:12,485 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=188, ProcessCount=11, AvailableMemoryMB=8893 2024-11-11T07:22:12,494 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:22:12,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c35b008 2024-11-11T07:22:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:22:12,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:22:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:22:12,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:22:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:12,522 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:22:12,542 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:12,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T07:22:12,546 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:22:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:22:12,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741835_1011 (size=959) 2024-11-11T07:22:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:22:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:22:12,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:22:12,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741836_1012 (size=53) 2024-11-11T07:22:12,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:12,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5a2be9895ea70c6a849961e6f35c48df, disabling compactions & flushes 2024-11-11T07:22:12,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:12,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:12,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. after waiting 0 ms 2024-11-11T07:22:12,977 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:12,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:12,978 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5a2be9895ea70c6a849961e6f35c48df: Waiting for close lock at 1731309732977Disabling compacts and flushes for region at 1731309732977Disabling writes for close at 1731309732977Writing region close event to WAL at 1731309732977Closed at 1731309732977 2024-11-11T07:22:12,980 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:22:12,985 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731309732980"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731309732980"}]},"ts":"1731309732980"} 2024-11-11T07:22:12,989 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:22:12,991 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:22:12,993 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309732991"}]},"ts":"1731309732991"} 2024-11-11T07:22:12,998 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:22:12,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, ASSIGN}] 2024-11-11T07:22:13,001 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, ASSIGN 2024-11-11T07:22:13,003 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, ASSIGN; state=OFFLINE, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=false 2024-11-11T07:22:13,155 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5a2be9895ea70c6a849961e6f35c48df, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:13,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, ASSIGN because future has completed 2024-11-11T07:22:13,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:22:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:22:13,321 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:13,321 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:22:13,322 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,322 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:13,322 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,322 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,325 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,327 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2be9895ea70c6a849961e6f35c48df columnFamilyName A 2024-11-11T07:22:13,327 DEBUG [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:13,328 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] regionserver.HStore(327): Store=5a2be9895ea70c6a849961e6f35c48df/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:13,329 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,331 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2be9895ea70c6a849961e6f35c48df columnFamilyName B 2024-11-11T07:22:13,331 DEBUG [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:13,331 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] regionserver.HStore(327): Store=5a2be9895ea70c6a849961e6f35c48df/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:13,332 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,334 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2be9895ea70c6a849961e6f35c48df columnFamilyName C 2024-11-11T07:22:13,334 DEBUG [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:13,334 INFO [StoreOpener-5a2be9895ea70c6a849961e6f35c48df-1 {}] regionserver.HStore(327): Store=5a2be9895ea70c6a849961e6f35c48df/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:13,335 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,337 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,337 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,340 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,340 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,341 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:22:13,344 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,348 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:22:13,349 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5a2be9895ea70c6a849961e6f35c48df; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71000749, jitterRate=0.057993605732917786}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:22:13,349 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,350 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5a2be9895ea70c6a849961e6f35c48df: Running coprocessor pre-open hook at 1731309733322Writing region info on filesystem at 1731309733322Initializing all the Stores at 1731309733324 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309733324Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309733324Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309733325 (+1 ms)Cleaning up temporary data from old regions at 1731309733340 (+15 ms)Running coprocessor post-open hooks at 1731309733349 (+9 ms)Region opened successfully at 1731309733350 (+1 ms) 2024-11-11T07:22:13,353 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., pid=6, masterSystemTime=1731309733314 2024-11-11T07:22:13,356 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:13,356 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:13,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5a2be9895ea70c6a849961e6f35c48df, regionState=OPEN, openSeqNum=2, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:13,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:22:13,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T07:22:13,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 in 203 msec 2024-11-11T07:22:13,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T07:22:13,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, ASSIGN in 368 msec 2024-11-11T07:22:13,374 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:22:13,374 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309733374"}]},"ts":"1731309733374"} 2024-11-11T07:22:13,377 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:22:13,379 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:22:13,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 865 msec 2024-11-11T07:22:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:22:13,693 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:13,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,700 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d463185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,704 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f0fa42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,706 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,713 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:22:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64ba8396 2024-11-11T07:22:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:22:13,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:22:13,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470128a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174557e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,734 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,738 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,741 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,745 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,753 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,758 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4331d0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,760 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17daddbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,766 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ad14c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,769 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6396285d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439f6406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,774 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92f3d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,776 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:13,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:13,779 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cba810e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:13,781 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c8f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:13,787 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,787 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,787 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,787 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,788 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,789 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,809 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:13,813 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,826 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:22:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T07:22:13,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:22:13,850 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:13,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:13,880 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:13,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:22:14,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/1affbadaee5c4ef88d916c59e17ce2af is 50, key is test_row_0/A:col10/1731309733866/Put/seqid=0 2024-11-11T07:22:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309794053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309794064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309794082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741837_1013 (size=14341) 2024-11-11T07:22:14,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309794086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/1affbadaee5c4ef88d916c59e17ce2af 2024-11-11T07:22:14,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309794087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:22:14,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/dc05cca9620f46daab7b1218d338a10d is 50, key is test_row_0/B:col10/1731309733866/Put/seqid=0 2024-11-11T07:22:14,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:14,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741838_1014 (size=12001) 2024-11-11T07:22:14,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/dc05cca9620f46daab7b1218d338a10d 2024-11-11T07:22:14,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309794234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309794234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309794236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309794238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309794239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/14b9492525e047789437254c35962bc4 is 50, key is test_row_0/C:col10/1731309733866/Put/seqid=0 2024-11-11T07:22:14,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741839_1015 (size=12001) 2024-11-11T07:22:14,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/14b9492525e047789437254c35962bc4 2024-11-11T07:22:14,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/1affbadaee5c4ef88d916c59e17ce2af as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/1affbadaee5c4ef88d916c59e17ce2af 2024-11-11T07:22:14,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/1affbadaee5c4ef88d916c59e17ce2af, entries=200, sequenceid=13, filesize=14.0 K 2024-11-11T07:22:14,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/dc05cca9620f46daab7b1218d338a10d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/dc05cca9620f46daab7b1218d338a10d 2024-11-11T07:22:14,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/dc05cca9620f46daab7b1218d338a10d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T07:22:14,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/14b9492525e047789437254c35962bc4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/14b9492525e047789437254c35962bc4 2024-11-11T07:22:14,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:14,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/14b9492525e047789437254c35962bc4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T07:22:14,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 5a2be9895ea70c6a849961e6f35c48df in 541ms, sequenceid=13, compaction requested=false 2024-11-11T07:22:14,429 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T07:22:14,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:14,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T07:22:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:22:14,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b668c16d28534e52be6e13f77d4aa0fc is 50, key is test_row_0/A:col10/1731309734446/Put/seqid=0 2024-11-11T07:22:14,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309794470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309794474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309794486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309794493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309794493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741840_1016 (size=14341) 2024-11-11T07:22:14,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b668c16d28534e52be6e13f77d4aa0fc 2024-11-11T07:22:14,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:14,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/bfb4d4af6ec5446b90f6679d56451c8c is 50, key is test_row_0/B:col10/1731309734446/Put/seqid=0 2024-11-11T07:22:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741841_1017 (size=12001) 2024-11-11T07:22:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309794604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/bfb4d4af6ec5446b90f6679d56451c8c 2024-11-11T07:22:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309794604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309794615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309794615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309794617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/35dcc4f4373a44f0b8c6350f4bca69bb is 50, key is test_row_0/C:col10/1731309734446/Put/seqid=0 2024-11-11T07:22:14,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741842_1018 (size=12001) 2024-11-11T07:22:14,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309794823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309794832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309794833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309794833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309794833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:14,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:14,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:22:15,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:15,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:15,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/35dcc4f4373a44f0b8c6350f4bca69bb 2024-11-11T07:22:15,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b668c16d28534e52be6e13f77d4aa0fc as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b668c16d28534e52be6e13f77d4aa0fc 2024-11-11T07:22:15,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b668c16d28534e52be6e13f77d4aa0fc, entries=200, sequenceid=40, filesize=14.0 K 2024-11-11T07:22:15,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/bfb4d4af6ec5446b90f6679d56451c8c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/bfb4d4af6ec5446b90f6679d56451c8c 2024-11-11T07:22:15,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309795133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/bfb4d4af6ec5446b90f6679d56451c8c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T07:22:15,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/35dcc4f4373a44f0b8c6350f4bca69bb as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/35dcc4f4373a44f0b8c6350f4bca69bb 2024-11-11T07:22:15,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309795142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/35dcc4f4373a44f0b8c6350f4bca69bb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T07:22:15,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 5a2be9895ea70c6a849961e6f35c48df in 704ms, sequenceid=40, compaction requested=false 2024-11-11T07:22:15,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:15,162 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:15,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:15,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c3e9f675087647a4a8fd1044547aeb18 is 50, key is test_row_0/A:col10/1731309735161/Put/seqid=0 2024-11-11T07:22:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741843_1019 (size=14341) 2024-11-11T07:22:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309795314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309795321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309795324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309795432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309795442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309795442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c3e9f675087647a4a8fd1044547aeb18 2024-11-11T07:22:15,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a6060d115317489b9affc3a2ef9d39fd is 50, key is test_row_0/B:col10/1731309735161/Put/seqid=0 2024-11-11T07:22:15,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309795642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309795651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309795662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309795662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309795662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741844_1020 (size=12001) 2024-11-11T07:22:15,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309795952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:15,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:15,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:15,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309795982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309795982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:22:16,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a6060d115317489b9affc3a2ef9d39fd 2024-11-11T07:22:16,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:16,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:16,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:16,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:16,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b7292c4fa3754ae28f0ecea07dc00363 is 50, key is test_row_0/C:col10/1731309735161/Put/seqid=0 2024-11-11T07:22:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741845_1021 (size=12001) 2024-11-11T07:22:16,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b7292c4fa3754ae28f0ecea07dc00363 2024-11-11T07:22:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c3e9f675087647a4a8fd1044547aeb18 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c3e9f675087647a4a8fd1044547aeb18 2024-11-11T07:22:16,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c3e9f675087647a4a8fd1044547aeb18, entries=200, sequenceid=52, filesize=14.0 K 2024-11-11T07:22:16,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a6060d115317489b9affc3a2ef9d39fd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a6060d115317489b9affc3a2ef9d39fd 2024-11-11T07:22:16,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a6060d115317489b9affc3a2ef9d39fd, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T07:22:16,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b7292c4fa3754ae28f0ecea07dc00363 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b7292c4fa3754ae28f0ecea07dc00363 2024-11-11T07:22:16,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b7292c4fa3754ae28f0ecea07dc00363, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T07:22:16,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5a2be9895ea70c6a849961e6f35c48df in 1075ms, sequenceid=52, compaction requested=true 2024-11-11T07:22:16,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:16,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:16,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:16,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:16,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:16,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:16,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:16,247 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:16,247 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:16,255 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:16,256 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:16,262 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:16,262 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:16,262 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:16,262 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:16,263 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/dc05cca9620f46daab7b1218d338a10d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/bfb4d4af6ec5446b90f6679d56451c8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a6060d115317489b9affc3a2ef9d39fd] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.2 K 2024-11-11T07:22:16,263 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/1affbadaee5c4ef88d916c59e17ce2af, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b668c16d28534e52be6e13f77d4aa0fc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c3e9f675087647a4a8fd1044547aeb18] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=42.0 K 2024-11-11T07:22:16,265 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting dc05cca9620f46daab7b1218d338a10d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731309733866 2024-11-11T07:22:16,266 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb4d4af6ec5446b90f6679d56451c8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731309734045 2024-11-11T07:22:16,266 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1affbadaee5c4ef88d916c59e17ce2af, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731309733865 2024-11-11T07:22:16,267 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a6060d115317489b9affc3a2ef9d39fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309734485 2024-11-11T07:22:16,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:22:16,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:16,268 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:22:16,269 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b668c16d28534e52be6e13f77d4aa0fc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731309734045 2024-11-11T07:22:16,270 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3e9f675087647a4a8fd1044547aeb18, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309734485 2024-11-11T07:22:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/892187065a854ff2b817ea9ab448a69d is 50, key is test_row_0/A:col10/1731309735320/Put/seqid=0 2024-11-11T07:22:16,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741846_1022 (size=12001) 2024-11-11T07:22:16,324 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/892187065a854ff2b817ea9ab448a69d 2024-11-11T07:22:16,331 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#11 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:16,332 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#10 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:16,333 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c49a4ee40e5b48aa8539e8772cd66fa0 is 50, key is test_row_0/A:col10/1731309735161/Put/seqid=0 2024-11-11T07:22:16,333 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/0621882202ee4ce28b4b0751aa2b753f is 50, key is test_row_0/B:col10/1731309735161/Put/seqid=0 2024-11-11T07:22:16,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741848_1024 (size=12104) 2024-11-11T07:22:16,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741847_1023 (size=12104) 2024-11-11T07:22:16,368 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c49a4ee40e5b48aa8539e8772cd66fa0 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c49a4ee40e5b48aa8539e8772cd66fa0 2024-11-11T07:22:16,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a4892ed7ab1f4216b540840b08a2dcbd is 50, key is test_row_0/B:col10/1731309735320/Put/seqid=0 2024-11-11T07:22:16,376 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/0621882202ee4ce28b4b0751aa2b753f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0621882202ee4ce28b4b0751aa2b753f 2024-11-11T07:22:16,392 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into c49a4ee40e5b48aa8539e8772cd66fa0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:16,392 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:16,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741849_1025 (size=12001) 2024-11-11T07:22:16,397 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309736239; duration=0sec 2024-11-11T07:22:16,398 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a4892ed7ab1f4216b540840b08a2dcbd 2024-11-11T07:22:16,398 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:16,398 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:16,398 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:16,401 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:16,402 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:16,402 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:16,402 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/14b9492525e047789437254c35962bc4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/35dcc4f4373a44f0b8c6350f4bca69bb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b7292c4fa3754ae28f0ecea07dc00363] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.2 K 2024-11-11T07:22:16,403 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14b9492525e047789437254c35962bc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731309733866 2024-11-11T07:22:16,405 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35dcc4f4373a44f0b8c6350f4bca69bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731309734045 2024-11-11T07:22:16,407 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7292c4fa3754ae28f0ecea07dc00363, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309734485 2024-11-11T07:22:16,417 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 0621882202ee4ce28b4b0751aa2b753f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:16,417 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:16,417 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309736243; duration=0sec 2024-11-11T07:22:16,417 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:16,417 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:16,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b0398632fbf5439fad7fd7218269b70f is 50, key is test_row_0/C:col10/1731309735320/Put/seqid=0 2024-11-11T07:22:16,447 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#14 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:16,450 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/7bc017ba53a341508780bd8c5fcfe2ff is 50, key is test_row_0/C:col10/1731309735161/Put/seqid=0 2024-11-11T07:22:16,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741850_1026 (size=12001) 2024-11-11T07:22:16,472 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b0398632fbf5439fad7fd7218269b70f 2024-11-11T07:22:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:16,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:16,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/892187065a854ff2b817ea9ab448a69d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/892187065a854ff2b817ea9ab448a69d 2024-11-11T07:22:16,501 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/892187065a854ff2b817ea9ab448a69d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T07:22:16,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a4892ed7ab1f4216b540840b08a2dcbd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a4892ed7ab1f4216b540840b08a2dcbd 2024-11-11T07:22:16,520 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a4892ed7ab1f4216b540840b08a2dcbd, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T07:22:16,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741851_1027 (size=12104) 2024-11-11T07:22:16,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b0398632fbf5439fad7fd7218269b70f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b0398632fbf5439fad7fd7218269b70f 2024-11-11T07:22:16,536 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/7bc017ba53a341508780bd8c5fcfe2ff as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7bc017ba53a341508780bd8c5fcfe2ff 2024-11-11T07:22:16,549 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into 7bc017ba53a341508780bd8c5fcfe2ff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:16,549 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:16,549 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309736243; duration=0sec 2024-11-11T07:22:16,549 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b0398632fbf5439fad7fd7218269b70f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T07:22:16,549 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:16,550 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:16,551 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=46.96 KB/48090 for 5a2be9895ea70c6a849961e6f35c48df in 284ms, sequenceid=76, compaction requested=false 2024-11-11T07:22:16,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:16,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:16,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T07:22:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T07:22:16,556 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:16,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:16,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T07:22:16,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7020 sec 2024-11-11T07:22:16,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7440 sec 2024-11-11T07:22:16,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/a1b2ddeaf18e4dcab4035f4919a9db82 is 50, key is test_row_0/A:col10/1731309736514/Put/seqid=0 2024-11-11T07:22:16,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741852_1028 (size=19021) 2024-11-11T07:22:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309796681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309796683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309796686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309796686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309796692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,700 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 5a2be9895ea70c6a849961e6f35c48df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309796801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309796802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309796802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309796802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:16,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:16,942 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T07:22:17,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/a1b2ddeaf18e4dcab4035f4919a9db82 2024-11-11T07:22:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309797022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309797023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309797023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309797023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/3d56e8bf6a0c460fa22b72d9b3f8619e is 50, key is test_row_0/B:col10/1731309736514/Put/seqid=0 2024-11-11T07:22:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741853_1029 (size=12001) 2024-11-11T07:22:17,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/3d56e8bf6a0c460fa22b72d9b3f8619e 2024-11-11T07:22:17,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/91b2e74cc3464671bdfdf808358d88d9 is 50, key is test_row_0/C:col10/1731309736514/Put/seqid=0 2024-11-11T07:22:17,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741854_1030 (size=12001) 2024-11-11T07:22:17,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/91b2e74cc3464671bdfdf808358d88d9 2024-11-11T07:22:17,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/a1b2ddeaf18e4dcab4035f4919a9db82 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a1b2ddeaf18e4dcab4035f4919a9db82 2024-11-11T07:22:17,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a1b2ddeaf18e4dcab4035f4919a9db82, entries=300, sequenceid=90, filesize=18.6 K 2024-11-11T07:22:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/3d56e8bf6a0c460fa22b72d9b3f8619e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/3d56e8bf6a0c460fa22b72d9b3f8619e 2024-11-11T07:22:17,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/3d56e8bf6a0c460fa22b72d9b3f8619e, entries=150, sequenceid=90, filesize=11.7 K 2024-11-11T07:22:17,165 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T07:22:17,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/91b2e74cc3464671bdfdf808358d88d9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/91b2e74cc3464671bdfdf808358d88d9 2024-11-11T07:22:17,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/91b2e74cc3464671bdfdf808358d88d9, entries=150, sequenceid=90, filesize=11.7 K 2024-11-11T07:22:17,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5a2be9895ea70c6a849961e6f35c48df in 624ms, sequenceid=90, compaction requested=true 2024-11-11T07:22:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:17,181 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:17,181 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:17,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:17,184 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:17,184 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:17,185 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:17,185 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c49a4ee40e5b48aa8539e8772cd66fa0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/892187065a854ff2b817ea9ab448a69d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a1b2ddeaf18e4dcab4035f4919a9db82] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=42.1 K 2024-11-11T07:22:17,186 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:17,186 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:17,186 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:17,186 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0621882202ee4ce28b4b0751aa2b753f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a4892ed7ab1f4216b540840b08a2dcbd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/3d56e8bf6a0c460fa22b72d9b3f8619e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.3 K 2024-11-11T07:22:17,187 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49a4ee40e5b48aa8539e8772cd66fa0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309734485 2024-11-11T07:22:17,187 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 0621882202ee4ce28b4b0751aa2b753f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309734485 2024-11-11T07:22:17,187 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892187065a854ff2b817ea9ab448a69d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731309735310 2024-11-11T07:22:17,188 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1b2ddeaf18e4dcab4035f4919a9db82, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309736472 2024-11-11T07:22:17,189 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a4892ed7ab1f4216b540840b08a2dcbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731309735310 2024-11-11T07:22:17,191 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d56e8bf6a0c460fa22b72d9b3f8619e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309736505 2024-11-11T07:22:17,232 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:17,233 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/756308c1e3d94ec3b177047c36d241d5 is 50, key is test_row_0/B:col10/1731309736514/Put/seqid=0 2024-11-11T07:22:17,236 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#18 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:17,237 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b02e3f0638404aca842d1b6df54978e6 is 50, key is test_row_0/A:col10/1731309736514/Put/seqid=0 2024-11-11T07:22:17,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741855_1031 (size=12207) 2024-11-11T07:22:17,284 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b02e3f0638404aca842d1b6df54978e6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b02e3f0638404aca842d1b6df54978e6 2024-11-11T07:22:17,301 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into b02e3f0638404aca842d1b6df54978e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:17,301 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:17,301 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309737181; duration=0sec 2024-11-11T07:22:17,302 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:17,302 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:17,302 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:17,306 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:17,306 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:17,307 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:17,307 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7bc017ba53a341508780bd8c5fcfe2ff, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b0398632fbf5439fad7fd7218269b70f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/91b2e74cc3464671bdfdf808358d88d9] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.3 K 2024-11-11T07:22:17,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741856_1032 (size=12207) 2024-11-11T07:22:17,312 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bc017ba53a341508780bd8c5fcfe2ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309734485 2024-11-11T07:22:17,320 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0398632fbf5439fad7fd7218269b70f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731309735310 2024-11-11T07:22:17,321 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b2e74cc3464671bdfdf808358d88d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309736505 2024-11-11T07:22:17,329 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/756308c1e3d94ec3b177047c36d241d5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/756308c1e3d94ec3b177047c36d241d5 2024-11-11T07:22:17,347 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:17,348 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:17,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T07:22:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:17,348 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/4bae24d2815248dcba8f84269d19cf76 is 50, key is test_row_0/C:col10/1731309736514/Put/seqid=0 2024-11-11T07:22:17,355 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 756308c1e3d94ec3b177047c36d241d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:17,355 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:17,355 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309737181; duration=0sec 2024-11-11T07:22:17,355 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:17,355 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:17,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/a5b30088eb7641c8adf249ed715cbe91 is 50, key is test_row_0/A:col10/1731309737343/Put/seqid=0 2024-11-11T07:22:17,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741857_1033 (size=12207) 2024-11-11T07:22:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309797360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309797362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309797364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309797369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741858_1034 (size=16681) 2024-11-11T07:22:17,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/a5b30088eb7641c8adf249ed715cbe91 2024-11-11T07:22:17,387 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/4bae24d2815248dcba8f84269d19cf76 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4bae24d2815248dcba8f84269d19cf76 2024-11-11T07:22:17,401 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into 4bae24d2815248dcba8f84269d19cf76(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:17,402 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:17,402 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309737181; duration=0sec 2024-11-11T07:22:17,402 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:17,402 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:17,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/49338511c2b44869a9a5c9ce500d059d is 50, key is test_row_0/B:col10/1731309737343/Put/seqid=0 2024-11-11T07:22:17,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741859_1035 (size=12001) 2024-11-11T07:22:17,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/49338511c2b44869a9a5c9ce500d059d 2024-11-11T07:22:17,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/310e90ea77b54f4e9df8a9179ec63bac is 50, key is test_row_0/C:col10/1731309737343/Put/seqid=0 2024-11-11T07:22:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741860_1036 (size=12001) 2024-11-11T07:22:17,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/310e90ea77b54f4e9df8a9179ec63bac 2024-11-11T07:22:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309797482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309797483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309797482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/a5b30088eb7641c8adf249ed715cbe91 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a5b30088eb7641c8adf249ed715cbe91 2024-11-11T07:22:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309797483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a5b30088eb7641c8adf249ed715cbe91, entries=250, sequenceid=118, filesize=16.3 K 2024-11-11T07:22:17,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/49338511c2b44869a9a5c9ce500d059d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49338511c2b44869a9a5c9ce500d059d 2024-11-11T07:22:17,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49338511c2b44869a9a5c9ce500d059d, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T07:22:17,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/310e90ea77b54f4e9df8a9179ec63bac as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/310e90ea77b54f4e9df8a9179ec63bac 2024-11-11T07:22:17,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/310e90ea77b54f4e9df8a9179ec63bac, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T07:22:17,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 5a2be9895ea70c6a849961e6f35c48df in 181ms, sequenceid=118, compaction requested=false 2024-11-11T07:22:17,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:17,677 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:22:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:17,705 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:17,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:17,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/cb655b115c0a42a789b52f00382b5867 is 50, key is test_row_0/A:col10/1731309737698/Put/seqid=0 2024-11-11T07:22:17,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741861_1037 (size=14441) 2024-11-11T07:22:17,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/cb655b115c0a42a789b52f00382b5867 2024-11-11T07:22:17,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309797765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/29248ab0cf6b4a95b906f0f495dd6c3f is 50, key is test_row_0/B:col10/1731309737698/Put/seqid=0 2024-11-11T07:22:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309797767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309797774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309797778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741862_1038 (size=12101) 2024-11-11T07:22:17,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309797881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T07:22:17,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T07:22:17,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T07:22:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T07:22:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309797892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T07:22:17,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T07:22:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309797892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T07:22:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,895 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T07:22:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309797892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:22:18,012 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:18,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T07:22:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:22:18,018 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:18,020 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:18,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:18,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309798092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309798102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309798102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309798103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:22:18,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T07:22:18,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:18,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:18,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:18,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/29248ab0cf6b4a95b906f0f495dd6c3f 2024-11-11T07:22:18,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/44cd84be721a40e6b4ca1bd3202b6d66 is 50, key is test_row_0/C:col10/1731309737698/Put/seqid=0 2024-11-11T07:22:18,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741863_1039 (size=12101) 2024-11-11T07:22:18,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/44cd84be721a40e6b4ca1bd3202b6d66 2024-11-11T07:22:18,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/cb655b115c0a42a789b52f00382b5867 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/cb655b115c0a42a789b52f00382b5867 2024-11-11T07:22:18,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/cb655b115c0a42a789b52f00382b5867, entries=200, sequenceid=132, filesize=14.1 K 2024-11-11T07:22:18,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/29248ab0cf6b4a95b906f0f495dd6c3f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/29248ab0cf6b4a95b906f0f495dd6c3f 2024-11-11T07:22:18,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/29248ab0cf6b4a95b906f0f495dd6c3f, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T07:22:18,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/44cd84be721a40e6b4ca1bd3202b6d66 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/44cd84be721a40e6b4ca1bd3202b6d66 2024-11-11T07:22:18,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/44cd84be721a40e6b4ca1bd3202b6d66, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T07:22:18,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5a2be9895ea70c6a849961e6f35c48df in 608ms, sequenceid=132, compaction requested=true 2024-11-11T07:22:18,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:18,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:18,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:18,314 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:18,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:18,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:18,314 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:18,317 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:18,317 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:18,317 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:18,317 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/756308c1e3d94ec3b177047c36d241d5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49338511c2b44869a9a5c9ce500d059d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/29248ab0cf6b4a95b906f0f495dd6c3f] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.5 K 2024-11-11T07:22:18,317 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:18,318 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:18,318 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:18,318 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b02e3f0638404aca842d1b6df54978e6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a5b30088eb7641c8adf249ed715cbe91, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/cb655b115c0a42a789b52f00382b5867] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=42.3 K 2024-11-11T07:22:18,319 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 756308c1e3d94ec3b177047c36d241d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309736505 2024-11-11T07:22:18,319 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b02e3f0638404aca842d1b6df54978e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309736505 2024-11-11T07:22:18,319 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 49338511c2b44869a9a5c9ce500d059d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731309736678 2024-11-11T07:22:18,320 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b30088eb7641c8adf249ed715cbe91, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731309736678 2024-11-11T07:22:18,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:18,321 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb655b115c0a42a789b52f00382b5867, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731309737353 2024-11-11T07:22:18,320 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 29248ab0cf6b4a95b906f0f495dd6c3f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731309737353 2024-11-11T07:22:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:22:18,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T07:22:18,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:18,337 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:22:18,341 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:18,342 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/956d865d94dd48f7b5806e835203049b is 50, key is test_row_0/A:col10/1731309737698/Put/seqid=0 2024-11-11T07:22:18,351 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:18,351 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/9433bc4b23bb423d8668a9db0cdc6bdf is 50, key is test_row_0/B:col10/1731309737698/Put/seqid=0 2024-11-11T07:22:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/63eaa9cd24dd449396873481277a1042 is 50, key is test_row_0/A:col10/1731309737764/Put/seqid=0 2024-11-11T07:22:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741864_1040 (size=12409) 2024-11-11T07:22:18,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741866_1042 (size=12151) 2024-11-11T07:22:18,396 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/63eaa9cd24dd449396873481277a1042 2024-11-11T07:22:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:18,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:18,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741865_1041 (size=12409) 2024-11-11T07:22:18,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/592da6ddfe3842228ddfa527fc89ec01 is 50, key is test_row_0/B:col10/1731309737764/Put/seqid=0 2024-11-11T07:22:18,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741867_1043 (size=12151) 2024-11-11T07:22:18,444 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/592da6ddfe3842228ddfa527fc89ec01 2024-11-11T07:22:18,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/a14781530ba24ff7b3becba907d663b5 is 50, key is test_row_0/C:col10/1731309737764/Put/seqid=0 2024-11-11T07:22:18,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741868_1044 (size=12151) 2024-11-11T07:22:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309798471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309798472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309798476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309798473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309798591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309798591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309798592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309798592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:22:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309798721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,727 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 5a2be9895ea70c6a849961e6f35c48df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,803 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/956d865d94dd48f7b5806e835203049b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/956d865d94dd48f7b5806e835203049b 2024-11-11T07:22:18,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309798801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309798801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309798802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309798802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:18,814 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into 956d865d94dd48f7b5806e835203049b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:18,814 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:18,814 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309738314; duration=0sec 2024-11-11T07:22:18,815 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:18,815 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:18,815 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:18,817 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:18,817 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:18,817 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:18,817 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4bae24d2815248dcba8f84269d19cf76, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/310e90ea77b54f4e9df8a9179ec63bac, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/44cd84be721a40e6b4ca1bd3202b6d66] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.5 K 2024-11-11T07:22:18,821 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bae24d2815248dcba8f84269d19cf76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309736505 2024-11-11T07:22:18,822 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 310e90ea77b54f4e9df8a9179ec63bac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731309736678 2024-11-11T07:22:18,823 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44cd84be721a40e6b4ca1bd3202b6d66, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731309737353 2024-11-11T07:22:18,831 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/9433bc4b23bb423d8668a9db0cdc6bdf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/9433bc4b23bb423d8668a9db0cdc6bdf 2024-11-11T07:22:18,847 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 9433bc4b23bb423d8668a9db0cdc6bdf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:18,847 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:18,847 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309738314; duration=0sec 2024-11-11T07:22:18,847 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:18,847 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:18,854 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:18,855 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/9d72aa7540b847829e9f55136662021f is 50, key is test_row_0/C:col10/1731309737698/Put/seqid=0 2024-11-11T07:22:18,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741869_1045 (size=12409) 2024-11-11T07:22:18,873 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/a14781530ba24ff7b3becba907d663b5 2024-11-11T07:22:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/63eaa9cd24dd449396873481277a1042 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/63eaa9cd24dd449396873481277a1042 2024-11-11T07:22:18,887 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/9d72aa7540b847829e9f55136662021f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9d72aa7540b847829e9f55136662021f 2024-11-11T07:22:18,895 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/63eaa9cd24dd449396873481277a1042, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:22:18,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/592da6ddfe3842228ddfa527fc89ec01 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/592da6ddfe3842228ddfa527fc89ec01 2024-11-11T07:22:18,900 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into 9d72aa7540b847829e9f55136662021f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:18,900 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:18,900 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309738314; duration=0sec 2024-11-11T07:22:18,900 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:18,900 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:18,904 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/592da6ddfe3842228ddfa527fc89ec01, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:22:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/a14781530ba24ff7b3becba907d663b5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a14781530ba24ff7b3becba907d663b5 2024-11-11T07:22:18,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a14781530ba24ff7b3becba907d663b5, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:22:18,919 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 5a2be9895ea70c6a849961e6f35c48df in 582ms, sequenceid=156, compaction requested=false 2024-11-11T07:22:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T07:22:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T07:22:18,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T07:22:18,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 900 msec 2024-11-11T07:22:18,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 916 msec 2024-11-11T07:22:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:19,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:19,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:22:19,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/7f13415c755c4b11bba9e6e85d55bd34 is 50, key is test_row_0/A:col10/1731309739112/Put/seqid=0 2024-11-11T07:22:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:22:19,155 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:19,156 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T07:22:19,160 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:22:19,162 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:19,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:19,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741870_1046 (size=12151) 2024-11-11T07:22:19,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/7f13415c755c4b11bba9e6e85d55bd34 2024-11-11T07:22:19,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309799167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309799170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309799171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/f7cde8959a7442daa876e7f321a49290 is 50, key is test_row_0/B:col10/1731309739112/Put/seqid=0 2024-11-11T07:22:19,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309799178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741871_1047 (size=12151) 2024-11-11T07:22:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:22:19,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309799281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309799292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309799292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309799293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:22:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:22:19,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:22:19,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309799492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309799502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309799502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309799511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/f7cde8959a7442daa876e7f321a49290 2024-11-11T07:22:19,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:22:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f4395c9376c04496b3c532b15239f4ac is 50, key is test_row_0/C:col10/1731309739112/Put/seqid=0 2024-11-11T07:22:19,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741872_1048 (size=12151) 2024-11-11T07:22:19,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:22:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:22:19,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309799802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309799811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309799812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309799822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:19,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:22:19,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:19,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:19,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:20,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f4395c9376c04496b3c532b15239f4ac 2024-11-11T07:22:20,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/7f13415c755c4b11bba9e6e85d55bd34 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/7f13415c755c4b11bba9e6e85d55bd34 2024-11-11T07:22:20,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/7f13415c755c4b11bba9e6e85d55bd34, entries=150, sequenceid=174, filesize=11.9 K 2024-11-11T07:22:20,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/f7cde8959a7442daa876e7f321a49290 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/f7cde8959a7442daa876e7f321a49290 2024-11-11T07:22:20,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:22:20,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:20,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:20,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/f7cde8959a7442daa876e7f321a49290, entries=150, sequenceid=174, filesize=11.9 K 2024-11-11T07:22:20,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f4395c9376c04496b3c532b15239f4ac as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f4395c9376c04496b3c532b15239f4ac 2024-11-11T07:22:20,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f4395c9376c04496b3c532b15239f4ac, entries=150, sequenceid=174, filesize=11.9 K 2024-11-11T07:22:20,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5a2be9895ea70c6a849961e6f35c48df in 1002ms, sequenceid=174, compaction requested=true 2024-11-11T07:22:20,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:20,117 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:20,119 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:20,119 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:20,119 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:20,119 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/956d865d94dd48f7b5806e835203049b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/63eaa9cd24dd449396873481277a1042, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/7f13415c755c4b11bba9e6e85d55bd34] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.9 K 2024-11-11T07:22:20,120 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 956d865d94dd48f7b5806e835203049b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731309737353 2024-11-11T07:22:20,120 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63eaa9cd24dd449396873481277a1042, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309737756 2024-11-11T07:22:20,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:20,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:20,121 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:20,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:20,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:20,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:20,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:20,123 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f13415c755c4b11bba9e6e85d55bd34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731309738426 2024-11-11T07:22:20,124 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:20,124 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:20,124 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:20,124 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/9433bc4b23bb423d8668a9db0cdc6bdf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/592da6ddfe3842228ddfa527fc89ec01, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/f7cde8959a7442daa876e7f321a49290] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.9 K 2024-11-11T07:22:20,125 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 9433bc4b23bb423d8668a9db0cdc6bdf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731309737353 2024-11-11T07:22:20,126 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 592da6ddfe3842228ddfa527fc89ec01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309737756 2024-11-11T07:22:20,126 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f7cde8959a7442daa876e7f321a49290, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731309738426 2024-11-11T07:22:20,136 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:20,136 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/06d4690251dc4d51ae873e2e4d4564ea is 50, key is test_row_0/A:col10/1731309739112/Put/seqid=0 2024-11-11T07:22:20,139 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:20,139 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/82a342918b964630be52b932e94f2e59 is 50, key is test_row_0/B:col10/1731309739112/Put/seqid=0 2024-11-11T07:22:20,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741874_1050 (size=12561) 2024-11-11T07:22:20,155 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/82a342918b964630be52b932e94f2e59 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/82a342918b964630be52b932e94f2e59 2024-11-11T07:22:20,170 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 82a342918b964630be52b932e94f2e59(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:20,170 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:20,170 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309740121; duration=0sec 2024-11-11T07:22:20,171 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:20,171 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:20,171 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:20,172 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:20,172 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:20,172 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:20,173 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9d72aa7540b847829e9f55136662021f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a14781530ba24ff7b3becba907d663b5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f4395c9376c04496b3c532b15239f4ac] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=35.9 K 2024-11-11T07:22:20,173 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d72aa7540b847829e9f55136662021f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731309737353 2024-11-11T07:22:20,174 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a14781530ba24ff7b3becba907d663b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309737756 2024-11-11T07:22:20,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741873_1049 (size=12561) 2024-11-11T07:22:20,175 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f4395c9376c04496b3c532b15239f4ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731309738426 2024-11-11T07:22:20,185 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/06d4690251dc4d51ae873e2e4d4564ea as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/06d4690251dc4d51ae873e2e4d4564ea 2024-11-11T07:22:20,189 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:20,190 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/cea3ab140ff64f8790db97b5d46e8691 is 50, key is test_row_0/C:col10/1731309739112/Put/seqid=0 2024-11-11T07:22:20,197 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into 06d4690251dc4d51ae873e2e4d4564ea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:20,197 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:20,197 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309740117; duration=0sec 2024-11-11T07:22:20,197 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:20,198 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:20,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741875_1051 (size=12561) 2024-11-11T07:22:20,214 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/cea3ab140ff64f8790db97b5d46e8691 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/cea3ab140ff64f8790db97b5d46e8691 2024-11-11T07:22:20,229 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into cea3ab140ff64f8790db97b5d46e8691(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:20,229 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:20,229 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309740121; duration=0sec 2024-11-11T07:22:20,229 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:20,229 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:20,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:22:20,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:20,248 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T07:22:20,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/d8401429138548449ea7335851d47095 is 50, key is test_row_0/A:col10/1731309739168/Put/seqid=0 2024-11-11T07:22:20,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741876_1052 (size=12151) 2024-11-11T07:22:20,286 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/d8401429138548449ea7335851d47095 2024-11-11T07:22:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:22:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/6aa12f1058b449b79a161b5c04cd104b is 50, key is test_row_0/B:col10/1731309739168/Put/seqid=0 2024-11-11T07:22:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:20,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:20,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741877_1053 (size=12151) 2024-11-11T07:22:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309800338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309800370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309800370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309800370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309800481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309800481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309800482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309800482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309800692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309800692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309800692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309800692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:20,724 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/6aa12f1058b449b79a161b5c04cd104b 2024-11-11T07:22:20,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/325f622680934044bf477167c368b1f6 is 50, key is test_row_0/C:col10/1731309739168/Put/seqid=0 2024-11-11T07:22:20,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741878_1054 (size=12151) 2024-11-11T07:22:20,765 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/325f622680934044bf477167c368b1f6 2024-11-11T07:22:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/d8401429138548449ea7335851d47095 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d8401429138548449ea7335851d47095 2024-11-11T07:22:20,789 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d8401429138548449ea7335851d47095, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T07:22:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/6aa12f1058b449b79a161b5c04cd104b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6aa12f1058b449b79a161b5c04cd104b 2024-11-11T07:22:20,797 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6aa12f1058b449b79a161b5c04cd104b, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T07:22:20,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/325f622680934044bf477167c368b1f6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/325f622680934044bf477167c368b1f6 2024-11-11T07:22:20,805 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/325f622680934044bf477167c368b1f6, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T07:22:20,808 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 5a2be9895ea70c6a849961e6f35c48df in 559ms, sequenceid=198, compaction requested=false 2024-11-11T07:22:20,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:20,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:20,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T07:22:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T07:22:20,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T07:22:20,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-11-11T07:22:20,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6580 sec 2024-11-11T07:22:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:21,005 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:21,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T07:22:21,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/67ac483929f4472f8ed9a270849cdd8d is 50, key is test_row_0/A:col10/1731309741005/Put/seqid=0 2024-11-11T07:22:21,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309801031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309801033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741879_1055 (size=12151) 2024-11-11T07:22:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309801036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309801035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309801142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309801142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309801151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309801152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:22:21,312 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:21,312 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T07:22:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:22:21,316 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:21,317 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:21,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:21,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309801352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309801352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309801361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309801361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:22:21,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/67ac483929f4472f8ed9a270849cdd8d 2024-11-11T07:22:21,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/7a068c05ba2b437e9e52340a709c5489 is 50, key is test_row_0/B:col10/1731309741005/Put/seqid=0 2024-11-11T07:22:21,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741880_1056 (size=12151) 2024-11-11T07:22:21,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/7a068c05ba2b437e9e52340a709c5489 2024-11-11T07:22:21,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T07:22:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/72863b2b5c7b4439abe5dcee6a935b3c is 50, key is test_row_0/C:col10/1731309741005/Put/seqid=0 2024-11-11T07:22:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741881_1057 (size=12151) 2024-11-11T07:22:21,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T07:22:21,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:21,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:22:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309801662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309801662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309801681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309801681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:21,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T07:22:21,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:21,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/72863b2b5c7b4439abe5dcee6a935b3c 2024-11-11T07:22:21,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/67ac483929f4472f8ed9a270849cdd8d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/67ac483929f4472f8ed9a270849cdd8d 2024-11-11T07:22:21,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/67ac483929f4472f8ed9a270849cdd8d, entries=150, sequenceid=217, filesize=11.9 K 2024-11-11T07:22:21,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/7a068c05ba2b437e9e52340a709c5489 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7a068c05ba2b437e9e52340a709c5489 2024-11-11T07:22:21,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7a068c05ba2b437e9e52340a709c5489, entries=150, sequenceid=217, filesize=11.9 K 2024-11-11T07:22:21,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/72863b2b5c7b4439abe5dcee6a935b3c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/72863b2b5c7b4439abe5dcee6a935b3c 2024-11-11T07:22:21,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T07:22:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:22:21,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:21,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/72863b2b5c7b4439abe5dcee6a935b3c, entries=150, sequenceid=217, filesize=11.9 K 2024-11-11T07:22:21,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5a2be9895ea70c6a849961e6f35c48df in 945ms, sequenceid=217, compaction requested=true 2024-11-11T07:22:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:21,950 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:21,950 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:21,952 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:21,952 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:21,952 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,953 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/06d4690251dc4d51ae873e2e4d4564ea, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d8401429138548449ea7335851d47095, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/67ac483929f4472f8ed9a270849cdd8d] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.0 K 2024-11-11T07:22:21,953 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:21,953 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:21,953 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:21,953 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d4690251dc4d51ae873e2e4d4564ea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731309738426 2024-11-11T07:22:21,953 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/82a342918b964630be52b932e94f2e59, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6aa12f1058b449b79a161b5c04cd104b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7a068c05ba2b437e9e52340a709c5489] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.0 K 2024-11-11T07:22:21,954 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8401429138548449ea7335851d47095, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731309739166 2024-11-11T07:22:21,955 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 82a342918b964630be52b932e94f2e59, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731309738426 2024-11-11T07:22:21,955 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ac483929f4472f8ed9a270849cdd8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731309740341 2024-11-11T07:22:21,956 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa12f1058b449b79a161b5c04cd104b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731309739166 2024-11-11T07:22:21,957 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a068c05ba2b437e9e52340a709c5489, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731309740341 2024-11-11T07:22:21,966 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:21,968 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/6f2a43cccfb14801b9f1fd8f8f27e312 is 50, key is test_row_0/A:col10/1731309741005/Put/seqid=0 2024-11-11T07:22:21,973 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#46 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:21,974 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/8815ccbe49da49789aa2e5a3a8c56766 is 50, key is test_row_0/B:col10/1731309741005/Put/seqid=0 2024-11-11T07:22:21,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741882_1058 (size=12663) 2024-11-11T07:22:21,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741883_1059 (size=12663) 2024-11-11T07:22:21,989 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/8815ccbe49da49789aa2e5a3a8c56766 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8815ccbe49da49789aa2e5a3a8c56766 2024-11-11T07:22:21,998 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 8815ccbe49da49789aa2e5a3a8c56766(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:21,998 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:21,998 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309741950; duration=0sec 2024-11-11T07:22:21,999 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:21,999 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:21,999 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:22,001 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:22,001 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:22,001 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:22,001 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/cea3ab140ff64f8790db97b5d46e8691, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/325f622680934044bf477167c368b1f6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/72863b2b5c7b4439abe5dcee6a935b3c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.0 K 2024-11-11T07:22:22,002 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cea3ab140ff64f8790db97b5d46e8691, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731309738426 2024-11-11T07:22:22,002 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 325f622680934044bf477167c368b1f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731309739166 2024-11-11T07:22:22,003 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 72863b2b5c7b4439abe5dcee6a935b3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731309740341 2024-11-11T07:22:22,018 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:22,019 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/abf9329f1b8c4010bc87ed676fc50248 is 50, key is test_row_0/C:col10/1731309741005/Put/seqid=0 2024-11-11T07:22:22,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741884_1060 (size=12663) 2024-11-11T07:22:22,036 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/abf9329f1b8c4010bc87ed676fc50248 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/abf9329f1b8c4010bc87ed676fc50248 2024-11-11T07:22:22,046 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into abf9329f1b8c4010bc87ed676fc50248(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:22,046 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:22,046 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309741950; duration=0sec 2024-11-11T07:22:22,047 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:22,047 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:22,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T07:22:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:22,097 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T07:22:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/d724d662dade4e8887a56c53babe1bee is 50, key is test_row_0/A:col10/1731309741032/Put/seqid=0 2024-11-11T07:22:22,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741885_1061 (size=12151) 2024-11-11T07:22:22,117 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/d724d662dade4e8887a56c53babe1bee 2024-11-11T07:22:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/1767e4af16ea482ca85825725422f795 is 50, key is test_row_0/B:col10/1731309741032/Put/seqid=0 2024-11-11T07:22:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741886_1062 (size=12151) 2024-11-11T07:22:22,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309802209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309802210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309802211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309802212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309802322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309802322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309802322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309802322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,386 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/6f2a43cccfb14801b9f1fd8f8f27e312 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/6f2a43cccfb14801b9f1fd8f8f27e312 2024-11-11T07:22:22,405 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into 6f2a43cccfb14801b9f1fd8f8f27e312(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:22,405 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:22,405 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309741950; duration=0sec 2024-11-11T07:22:22,405 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:22,405 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:22:22,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309802532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309802532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309802533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309802533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/1767e4af16ea482ca85825725422f795 2024-11-11T07:22:22,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f0b3db212fa74ca590ac54e7c6acf8a9 is 50, key is test_row_0/C:col10/1731309741032/Put/seqid=0 2024-11-11T07:22:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741887_1063 (size=12151) 2024-11-11T07:22:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309802752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,755 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 5a2be9895ea70c6a849961e6f35c48df of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309802841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309802842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309802842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309802842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:22,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f0b3db212fa74ca590ac54e7c6acf8a9 2024-11-11T07:22:23,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/d724d662dade4e8887a56c53babe1bee as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d724d662dade4e8887a56c53babe1bee 2024-11-11T07:22:23,012 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d724d662dade4e8887a56c53babe1bee, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T07:22:23,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/1767e4af16ea482ca85825725422f795 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1767e4af16ea482ca85825725422f795 2024-11-11T07:22:23,028 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1767e4af16ea482ca85825725422f795, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T07:22:23,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f0b3db212fa74ca590ac54e7c6acf8a9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f0b3db212fa74ca590ac54e7c6acf8a9 2024-11-11T07:22:23,035 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f0b3db212fa74ca590ac54e7c6acf8a9, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T07:22:23,037 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5a2be9895ea70c6a849961e6f35c48df in 940ms, sequenceid=236, compaction requested=false 2024-11-11T07:22:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T07:22:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T07:22:23,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T07:22:23,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-11-11T07:22:23,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7300 sec 2024-11-11T07:22:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:23,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T07:22:23,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/58cd5c9a387744d688b19dad70907a4c is 50, key is test_row_0/A:col10/1731309743353/Put/seqid=0 2024-11-11T07:22:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309803375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309803377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309803379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309803385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741888_1064 (size=17031) 2024-11-11T07:22:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:22:23,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:23,462 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T07:22:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:22:23,466 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:23,467 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:23,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:23,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309803492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309803492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309803492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309803501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:22:23,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:23,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:23,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:23,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:23,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309803702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309803702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309803702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309803711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:23,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:23,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:23,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:23,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:23,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:22:23,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/58cd5c9a387744d688b19dad70907a4c 2024-11-11T07:22:23,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/50682463056547b5a8883aeab461eedc is 50, key is test_row_0/B:col10/1731309743353/Put/seqid=0 2024-11-11T07:22:23,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741889_1065 (size=12251) 2024-11-11T07:22:23,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:23,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:23,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309804012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309804012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309804022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309804022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:24,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:24,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:22:24,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/50682463056547b5a8883aeab461eedc 2024-11-11T07:22:24,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f98d8303b8864b2cb25cc8834fbc630f is 50, key is test_row_0/C:col10/1731309743353/Put/seqid=0 2024-11-11T07:22:24,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741890_1066 (size=12251) 2024-11-11T07:22:24,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:24,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:24,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309804531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309804532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309804532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309804532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:24,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:22:24,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f98d8303b8864b2cb25cc8834fbc630f 2024-11-11T07:22:24,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/58cd5c9a387744d688b19dad70907a4c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/58cd5c9a387744d688b19dad70907a4c 2024-11-11T07:22:24,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/58cd5c9a387744d688b19dad70907a4c, entries=250, sequenceid=259, filesize=16.6 K 2024-11-11T07:22:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/50682463056547b5a8883aeab461eedc as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/50682463056547b5a8883aeab461eedc 2024-11-11T07:22:24,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/50682463056547b5a8883aeab461eedc, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T07:22:24,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f98d8303b8864b2cb25cc8834fbc630f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f98d8303b8864b2cb25cc8834fbc630f 2024-11-11T07:22:24,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f98d8303b8864b2cb25cc8834fbc630f, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T07:22:24,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5a2be9895ea70c6a849961e6f35c48df in 1324ms, sequenceid=259, compaction requested=true 2024-11-11T07:22:24,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:24,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:24,678 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:24,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:24,679 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:24,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:24,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:24,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:24,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:24,680 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41845 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:24,680 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:24,680 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:24,680 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:24,680 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,680 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,680 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/6f2a43cccfb14801b9f1fd8f8f27e312, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d724d662dade4e8887a56c53babe1bee, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/58cd5c9a387744d688b19dad70907a4c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=40.9 K 2024-11-11T07:22:24,680 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8815ccbe49da49789aa2e5a3a8c56766, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1767e4af16ea482ca85825725422f795, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/50682463056547b5a8883aeab461eedc] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.2 K 2024-11-11T07:22:24,682 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f2a43cccfb14801b9f1fd8f8f27e312, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731309740341 2024-11-11T07:22:24,684 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8815ccbe49da49789aa2e5a3a8c56766, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731309740341 2024-11-11T07:22:24,686 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting d724d662dade4e8887a56c53babe1bee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731309741030 2024-11-11T07:22:24,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1767e4af16ea482ca85825725422f795, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731309741030 2024-11-11T07:22:24,686 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58cd5c9a387744d688b19dad70907a4c, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731309742195 2024-11-11T07:22:24,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 50682463056547b5a8883aeab461eedc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731309742207 2024-11-11T07:22:24,703 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:24,704 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ec05240400d44ae3a2daff9fd049f84a is 50, key is test_row_0/A:col10/1731309743353/Put/seqid=0 2024-11-11T07:22:24,710 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:24,711 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/6d5ab97ae92f425fb217b2aed7872855 is 50, key is test_row_0/B:col10/1731309743353/Put/seqid=0 2024-11-11T07:22:24,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:22:24,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,727 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:22:24,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741891_1067 (size=12865) 2024-11-11T07:22:24,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b75bcf69aa32498bb03f221e1d417e85 is 50, key is test_row_0/A:col10/1731309743369/Put/seqid=0 2024-11-11T07:22:24,741 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ec05240400d44ae3a2daff9fd049f84a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ec05240400d44ae3a2daff9fd049f84a 2024-11-11T07:22:24,752 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into ec05240400d44ae3a2daff9fd049f84a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:24,752 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:24,752 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309744678; duration=0sec 2024-11-11T07:22:24,752 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:24,752 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:24,752 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:24,754 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:24,754 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:24,754 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,754 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/abf9329f1b8c4010bc87ed676fc50248, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f0b3db212fa74ca590ac54e7c6acf8a9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f98d8303b8864b2cb25cc8834fbc630f] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.2 K 2024-11-11T07:22:24,755 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting abf9329f1b8c4010bc87ed676fc50248, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731309740341 2024-11-11T07:22:24,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741892_1068 (size=12865) 2024-11-11T07:22:24,756 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0b3db212fa74ca590ac54e7c6acf8a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731309741030 2024-11-11T07:22:24,757 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting f98d8303b8864b2cb25cc8834fbc630f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731309742207 2024-11-11T07:22:24,785 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:24,785 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f85be4ef0e21433083daa79ccf7f37e6 is 50, key is test_row_0/C:col10/1731309743353/Put/seqid=0 2024-11-11T07:22:24,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741893_1069 (size=12301) 2024-11-11T07:22:24,788 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b75bcf69aa32498bb03f221e1d417e85 2024-11-11T07:22:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/1371aa06186b4261b24cc48a38be6531 is 50, key is test_row_0/B:col10/1731309743369/Put/seqid=0 2024-11-11T07:22:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741894_1070 (size=12865) 2024-11-11T07:22:24,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741895_1071 (size=12301) 2024-11-11T07:22:24,850 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/1371aa06186b4261b24cc48a38be6531 2024-11-11T07:22:24,853 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/f85be4ef0e21433083daa79ccf7f37e6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f85be4ef0e21433083daa79ccf7f37e6 2024-11-11T07:22:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b14724340dc04cd8a632a128007cc8b9 is 50, key is test_row_0/C:col10/1731309743369/Put/seqid=0 2024-11-11T07:22:24,870 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into f85be4ef0e21433083daa79ccf7f37e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:24,871 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:24,871 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309744679; duration=0sec 2024-11-11T07:22:24,871 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:24,871 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:24,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741896_1072 (size=12301) 2024-11-11T07:22:24,899 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b14724340dc04cd8a632a128007cc8b9 2024-11-11T07:22:24,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/b75bcf69aa32498bb03f221e1d417e85 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b75bcf69aa32498bb03f221e1d417e85 2024-11-11T07:22:24,914 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b75bcf69aa32498bb03f221e1d417e85, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T07:22:24,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/1371aa06186b4261b24cc48a38be6531 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1371aa06186b4261b24cc48a38be6531 2024-11-11T07:22:24,933 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1371aa06186b4261b24cc48a38be6531, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T07:22:24,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b14724340dc04cd8a632a128007cc8b9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b14724340dc04cd8a632a128007cc8b9 2024-11-11T07:22:24,941 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b14724340dc04cd8a632a128007cc8b9, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T07:22:24,943 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 5a2be9895ea70c6a849961e6f35c48df in 215ms, sequenceid=273, compaction requested=false 2024-11-11T07:22:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T07:22:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T07:22:24,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T07:22:24,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4810 sec 2024-11-11T07:22:24,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4890 sec 2024-11-11T07:22:25,178 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/6d5ab97ae92f425fb217b2aed7872855 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6d5ab97ae92f425fb217b2aed7872855 2024-11-11T07:22:25,192 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 6d5ab97ae92f425fb217b2aed7872855(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:25,193 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:25,193 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309744679; duration=0sec 2024-11-11T07:22:25,193 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:25,193 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:25,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:25,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:25,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/96b019ed72064473b25c634f20ce76a2 is 50, key is test_row_0/A:col10/1731309745545/Put/seqid=0 2024-11-11T07:22:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:22:25,612 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:25,613 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T07:22:25,615 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:22:25,616 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:25,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:25,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741897_1073 (size=14741) 2024-11-11T07:22:25,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/96b019ed72064473b25c634f20ce76a2 2024-11-11T07:22:25,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309805641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309805637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309805650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309805651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2671476d274544899ee102aefc00f5cb is 50, key is test_row_0/B:col10/1731309745545/Put/seqid=0 2024-11-11T07:22:25,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741898_1074 (size=12301) 2024-11-11T07:22:25,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2671476d274544899ee102aefc00f5cb 2024-11-11T07:22:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:22:25,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/9943559c00cb462486f969082100d955 is 50, key is test_row_0/C:col10/1731309745545/Put/seqid=0 2024-11-11T07:22:25,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309805752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309805752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309805762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309805761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741899_1075 (size=12301) 2024-11-11T07:22:25,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:22:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:25,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:25,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:22:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:25,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:25,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:22:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309805962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309805972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309805972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:25,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309805972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:22:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:26,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:26,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/9943559c00cb462486f969082100d955 2024-11-11T07:22:26,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/96b019ed72064473b25c634f20ce76a2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/96b019ed72064473b25c634f20ce76a2 2024-11-11T07:22:26,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/96b019ed72064473b25c634f20ce76a2, entries=200, sequenceid=287, filesize=14.4 K 2024-11-11T07:22:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2671476d274544899ee102aefc00f5cb as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2671476d274544899ee102aefc00f5cb 2024-11-11T07:22:26,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2671476d274544899ee102aefc00f5cb, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T07:22:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/9943559c00cb462486f969082100d955 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9943559c00cb462486f969082100d955 2024-11-11T07:22:26,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9943559c00cb462486f969082100d955, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T07:22:26,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5a2be9895ea70c6a849961e6f35c48df in 642ms, sequenceid=287, compaction requested=true 2024-11-11T07:22:26,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:26,211 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:26,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:26,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:26,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:26,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:26,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:26,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:22:26,213 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:26,213 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:26,213 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:26,213 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:26,213 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ec05240400d44ae3a2daff9fd049f84a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b75bcf69aa32498bb03f221e1d417e85, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/96b019ed72064473b25c634f20ce76a2] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=39.0 K 2024-11-11T07:22:26,214 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec05240400d44ae3a2daff9fd049f84a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731309742207 2024-11-11T07:22:26,214 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75bcf69aa32498bb03f221e1d417e85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309743369 2024-11-11T07:22:26,215 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:26,216 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:26,216 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:26,216 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6d5ab97ae92f425fb217b2aed7872855, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1371aa06186b4261b24cc48a38be6531, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2671476d274544899ee102aefc00f5cb] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.6 K 2024-11-11T07:22:26,216 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b019ed72064473b25c634f20ce76a2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309745545 2024-11-11T07:22:26,217 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5ab97ae92f425fb217b2aed7872855, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731309742207 2024-11-11T07:22:26,218 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1371aa06186b4261b24cc48a38be6531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309743369 2024-11-11T07:22:26,218 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2671476d274544899ee102aefc00f5cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309745545 2024-11-11T07:22:26,239 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:26,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:22:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:26,240 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:22:26,240 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/eddfed098f5c45d797a54789e85efc3b is 50, key is test_row_0/A:col10/1731309745545/Put/seqid=0 2024-11-11T07:22:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:22:26,255 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:26,256 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/0e5490400e9d48cf8d2bf7694b3d11d1 is 50, key is test_row_0/B:col10/1731309745545/Put/seqid=0 2024-11-11T07:22:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ddc33e3475984078ae37ff06247962b9 is 50, key is test_row_0/A:col10/1731309745641/Put/seqid=0 2024-11-11T07:22:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:26,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:26,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309806300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309806302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309806302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741900_1076 (size=13017) 2024-11-11T07:22:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309806304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741901_1077 (size=13017) 2024-11-11T07:22:26,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741902_1078 (size=12301) 2024-11-11T07:22:26,325 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ddc33e3475984078ae37ff06247962b9 2024-11-11T07:22:26,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/033a20ca01434563b191ceeff58a6a44 is 50, key is test_row_0/B:col10/1731309745641/Put/seqid=0 2024-11-11T07:22:26,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741903_1079 (size=12301) 2024-11-11T07:22:26,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309806412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309806412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309806421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309806421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309806622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309806622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309806631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309806632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,724 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/eddfed098f5c45d797a54789e85efc3b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/eddfed098f5c45d797a54789e85efc3b 2024-11-11T07:22:26,725 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/0e5490400e9d48cf8d2bf7694b3d11d1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0e5490400e9d48cf8d2bf7694b3d11d1 2024-11-11T07:22:26,733 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into eddfed098f5c45d797a54789e85efc3b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:26,733 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 0e5490400e9d48cf8d2bf7694b3d11d1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:26,733 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:26,733 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:26,733 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309746212; duration=0sec 2024-11-11T07:22:26,733 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309746211; duration=0sec 2024-11-11T07:22:26,733 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:26,733 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:26,733 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:26,733 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:26,733 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:26,735 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:26,735 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:26,735 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:26,735 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f85be4ef0e21433083daa79ccf7f37e6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b14724340dc04cd8a632a128007cc8b9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9943559c00cb462486f969082100d955] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.6 K 2024-11-11T07:22:26,736 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f85be4ef0e21433083daa79ccf7f37e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731309742207 2024-11-11T07:22:26,736 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b14724340dc04cd8a632a128007cc8b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309743369 2024-11-11T07:22:26,737 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 9943559c00cb462486f969082100d955, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309745545 2024-11-11T07:22:26,750 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:26,751 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/033a20ca01434563b191ceeff58a6a44 2024-11-11T07:22:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:22:26,752 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/4fbf2ac53d7a4f209d47d25ddc379772 is 50, key is test_row_0/C:col10/1731309745545/Put/seqid=0 2024-11-11T07:22:26,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741904_1080 (size=13017) 2024-11-11T07:22:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/4eb3b07feef74466839135a03caf4398 is 50, key is test_row_0/C:col10/1731309745641/Put/seqid=0 2024-11-11T07:22:26,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741905_1081 (size=12301) 2024-11-11T07:22:26,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309806932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309806942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309806942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309806943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,170 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/4fbf2ac53d7a4f209d47d25ddc379772 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4fbf2ac53d7a4f209d47d25ddc379772 2024-11-11T07:22:27,179 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into 4fbf2ac53d7a4f209d47d25ddc379772(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:27,179 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:27,179 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309746212; duration=0sec 2024-11-11T07:22:27,179 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:27,179 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:27,207 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/4eb3b07feef74466839135a03caf4398 2024-11-11T07:22:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ddc33e3475984078ae37ff06247962b9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ddc33e3475984078ae37ff06247962b9 2024-11-11T07:22:27,225 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ddc33e3475984078ae37ff06247962b9, entries=150, sequenceid=312, filesize=12.0 K 2024-11-11T07:22:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/033a20ca01434563b191ceeff58a6a44 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/033a20ca01434563b191ceeff58a6a44 2024-11-11T07:22:27,232 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/033a20ca01434563b191ceeff58a6a44, entries=150, sequenceid=312, filesize=12.0 K 2024-11-11T07:22:27,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/4eb3b07feef74466839135a03caf4398 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4eb3b07feef74466839135a03caf4398 2024-11-11T07:22:27,239 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4eb3b07feef74466839135a03caf4398, entries=150, sequenceid=312, filesize=12.0 K 2024-11-11T07:22:27,240 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5a2be9895ea70c6a849961e6f35c48df in 1001ms, sequenceid=312, compaction requested=false 2024-11-11T07:22:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T07:22:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T07:22:27,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T07:22:27,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6250 sec 2024-11-11T07:22:27,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6330 sec 2024-11-11T07:22:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:27,445 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:27,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:22:27,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/00fcd316daf146d3b2a874a8da567ee1 is 50, key is test_row_0/A:col10/1731309747443/Put/seqid=0 2024-11-11T07:22:27,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741906_1082 (size=12301) 2024-11-11T07:22:27,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/00fcd316daf146d3b2a874a8da567ee1 2024-11-11T07:22:27,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309807482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/737c2e5c55a5496c91d35fdbd2e99abf is 50, key is test_row_0/B:col10/1731309747443/Put/seqid=0 2024-11-11T07:22:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309807484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309807485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309807487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741907_1083 (size=12301) 2024-11-11T07:22:27,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/737c2e5c55a5496c91d35fdbd2e99abf 2024-11-11T07:22:27,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/13995f07c4d6474e9f541dddab5ba6f4 is 50, key is test_row_0/C:col10/1731309747443/Put/seqid=0 2024-11-11T07:22:27,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741908_1084 (size=12301) 2024-11-11T07:22:27,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/13995f07c4d6474e9f541dddab5ba6f4 2024-11-11T07:22:27,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/00fcd316daf146d3b2a874a8da567ee1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/00fcd316daf146d3b2a874a8da567ee1 2024-11-11T07:22:27,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/00fcd316daf146d3b2a874a8da567ee1, entries=150, sequenceid=327, filesize=12.0 K 2024-11-11T07:22:27,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/737c2e5c55a5496c91d35fdbd2e99abf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/737c2e5c55a5496c91d35fdbd2e99abf 2024-11-11T07:22:27,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/737c2e5c55a5496c91d35fdbd2e99abf, entries=150, sequenceid=327, filesize=12.0 K 2024-11-11T07:22:27,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/13995f07c4d6474e9f541dddab5ba6f4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/13995f07c4d6474e9f541dddab5ba6f4 2024-11-11T07:22:27,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/13995f07c4d6474e9f541dddab5ba6f4, entries=150, sequenceid=327, filesize=12.0 K 2024-11-11T07:22:27,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5a2be9895ea70c6a849961e6f35c48df in 123ms, sequenceid=327, compaction requested=true 2024-11-11T07:22:27,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:27,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:27,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:27,569 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:27,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:27,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:27,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:27,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:27,569 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:27,571 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:27,571 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:27,571 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:27,571 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/eddfed098f5c45d797a54789e85efc3b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ddc33e3475984078ae37ff06247962b9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/00fcd316daf146d3b2a874a8da567ee1] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.7 K 2024-11-11T07:22:27,571 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:27,572 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:27,572 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:27,572 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0e5490400e9d48cf8d2bf7694b3d11d1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/033a20ca01434563b191ceeff58a6a44, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/737c2e5c55a5496c91d35fdbd2e99abf] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.7 K 2024-11-11T07:22:27,572 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5490400e9d48cf8d2bf7694b3d11d1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309745545 2024-11-11T07:22:27,573 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting eddfed098f5c45d797a54789e85efc3b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309745545 2024-11-11T07:22:27,573 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 033a20ca01434563b191ceeff58a6a44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731309745603 2024-11-11T07:22:27,573 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc33e3475984078ae37ff06247962b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731309745603 2024-11-11T07:22:27,574 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 737c2e5c55a5496c91d35fdbd2e99abf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731309746298 2024-11-11T07:22:27,574 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00fcd316daf146d3b2a874a8da567ee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731309746298 2024-11-11T07:22:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:27,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:27,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:22:27,599 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:27,600 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/e2bb6957cb7e4efbae4d2f6aa6bd6307 is 50, key is test_row_0/B:col10/1731309747443/Put/seqid=0 2024-11-11T07:22:27,601 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:27,602 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/fd6a8fe798814cb296926c8374f2d80b is 50, key is test_row_0/A:col10/1731309747443/Put/seqid=0 2024-11-11T07:22:27,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ba1f5b47f0a4416e97ec7af9ee7df487 is 50, key is test_row_0/A:col10/1731309747593/Put/seqid=0 2024-11-11T07:22:27,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309807613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309807614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309807617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309807618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741909_1085 (size=13119) 2024-11-11T07:22:27,644 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/fd6a8fe798814cb296926c8374f2d80b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/fd6a8fe798814cb296926c8374f2d80b 2024-11-11T07:22:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741910_1086 (size=13119) 2024-11-11T07:22:27,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741911_1087 (size=14741) 2024-11-11T07:22:27,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ba1f5b47f0a4416e97ec7af9ee7df487 2024-11-11T07:22:27,654 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into fd6a8fe798814cb296926c8374f2d80b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:27,654 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:27,654 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309747569; duration=0sec 2024-11-11T07:22:27,654 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:27,655 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:27,655 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:27,657 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:27,657 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:27,657 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:27,658 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4fbf2ac53d7a4f209d47d25ddc379772, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4eb3b07feef74466839135a03caf4398, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/13995f07c4d6474e9f541dddab5ba6f4] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.7 K 2024-11-11T07:22:27,660 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbf2ac53d7a4f209d47d25ddc379772, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309745545 2024-11-11T07:22:27,660 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb3b07feef74466839135a03caf4398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731309745603 2024-11-11T07:22:27,661 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13995f07c4d6474e9f541dddab5ba6f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731309746298 2024-11-11T07:22:27,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/720bc1f033314766b66b25b6346e3104 is 50, key is test_row_0/B:col10/1731309747593/Put/seqid=0 2024-11-11T07:22:27,678 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:27,678 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/de3b3f94d0ab413884c008353844a068 is 50, key is test_row_0/C:col10/1731309747443/Put/seqid=0 2024-11-11T07:22:27,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741912_1088 (size=12301) 2024-11-11T07:22:27,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/720bc1f033314766b66b25b6346e3104 2024-11-11T07:22:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741913_1089 (size=13119) 2024-11-11T07:22:27,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309807722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309807722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,728 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/de3b3f94d0ab413884c008353844a068 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/de3b3f94d0ab413884c008353844a068 2024-11-11T07:22:27,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/303f5a32862244ce8a0b1d7a2a9e5b88 is 50, key is test_row_0/C:col10/1731309747593/Put/seqid=0 2024-11-11T07:22:27,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309807733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309807733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,761 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into de3b3f94d0ab413884c008353844a068(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:27,761 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:27,762 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309747569; duration=0sec 2024-11-11T07:22:27,762 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:27,762 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:27,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741914_1090 (size=12301) 2024-11-11T07:22:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:22:27,772 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:27,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T07:22:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:22:27,778 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:27,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:27,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:22:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309807932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309807933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:27,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:27,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:27,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:27,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:27,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:27,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309807961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:27,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309807962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:27,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,056 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/e2bb6957cb7e4efbae4d2f6aa6bd6307 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e2bb6957cb7e4efbae4d2f6aa6bd6307 2024-11-11T07:22:28,062 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into e2bb6957cb7e4efbae4d2f6aa6bd6307(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:28,062 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:28,062 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309747569; duration=0sec 2024-11-11T07:22:28,063 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:28,063 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:22:28,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/303f5a32862244ce8a0b1d7a2a9e5b88 2024-11-11T07:22:28,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/ba1f5b47f0a4416e97ec7af9ee7df487 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ba1f5b47f0a4416e97ec7af9ee7df487 2024-11-11T07:22:28,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ba1f5b47f0a4416e97ec7af9ee7df487, entries=200, sequenceid=352, filesize=14.4 K 2024-11-11T07:22:28,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/720bc1f033314766b66b25b6346e3104 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/720bc1f033314766b66b25b6346e3104 2024-11-11T07:22:28,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/720bc1f033314766b66b25b6346e3104, entries=150, sequenceid=352, filesize=12.0 K 2024-11-11T07:22:28,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/303f5a32862244ce8a0b1d7a2a9e5b88 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/303f5a32862244ce8a0b1d7a2a9e5b88 2024-11-11T07:22:28,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/303f5a32862244ce8a0b1d7a2a9e5b88, entries=150, sequenceid=352, filesize=12.0 K 2024-11-11T07:22:28,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5a2be9895ea70c6a849961e6f35c48df in 639ms, sequenceid=352, compaction requested=false 2024-11-11T07:22:28,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:28,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:22:28,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/8ee901abe61f4eb192bd7bb482097ba5 is 50, key is test_row_0/A:col10/1731309747616/Put/seqid=0 2024-11-11T07:22:28,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741915_1091 (size=14741) 2024-11-11T07:22:28,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309808286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309808288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309808289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309808291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:22:28,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309808401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309808401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309808402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309808402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:28,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309808612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309808612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309808612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309808612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/8ee901abe61f4eb192bd7bb482097ba5 2024-11-11T07:22:28,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2c9a35f00f02423781d75ec0767c80b8 is 50, key is test_row_0/B:col10/1731309747616/Put/seqid=0 2024-11-11T07:22:28,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741916_1092 (size=12301) 2024-11-11T07:22:28,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2c9a35f00f02423781d75ec0767c80b8 2024-11-11T07:22:28,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/c5e07633d2d44957b2a75bd99fcf40cf is 50, key is test_row_0/C:col10/1731309747616/Put/seqid=0 2024-11-11T07:22:28,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741917_1093 (size=12301) 2024-11-11T07:22:28,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:28,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:22:28,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309808922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309808922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309808922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309808923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:29,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/c5e07633d2d44957b2a75bd99fcf40cf 2024-11-11T07:22:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/8ee901abe61f4eb192bd7bb482097ba5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/8ee901abe61f4eb192bd7bb482097ba5 2024-11-11T07:22:29,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/8ee901abe61f4eb192bd7bb482097ba5, entries=200, sequenceid=368, filesize=14.4 K 2024-11-11T07:22:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2c9a35f00f02423781d75ec0767c80b8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2c9a35f00f02423781d75ec0767c80b8 2024-11-11T07:22:29,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:29,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:29,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2c9a35f00f02423781d75ec0767c80b8, entries=150, sequenceid=368, filesize=12.0 K 2024-11-11T07:22:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:29,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/c5e07633d2d44957b2a75bd99fcf40cf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c5e07633d2d44957b2a75bd99fcf40cf 2024-11-11T07:22:29,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c5e07633d2d44957b2a75bd99fcf40cf, entries=150, sequenceid=368, filesize=12.0 K 2024-11-11T07:22:29,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5a2be9895ea70c6a849961e6f35c48df in 953ms, sequenceid=368, compaction requested=true 2024-11-11T07:22:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:29,197 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:29,197 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:29,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:29,198 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:29,198 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:29,198 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:29,198 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:29,199 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,199 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,199 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e2bb6957cb7e4efbae4d2f6aa6bd6307, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/720bc1f033314766b66b25b6346e3104, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2c9a35f00f02423781d75ec0767c80b8] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.8 K 2024-11-11T07:22:29,199 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/fd6a8fe798814cb296926c8374f2d80b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ba1f5b47f0a4416e97ec7af9ee7df487, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/8ee901abe61f4eb192bd7bb482097ba5] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=41.6 K 2024-11-11T07:22:29,199 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e2bb6957cb7e4efbae4d2f6aa6bd6307, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731309746298 2024-11-11T07:22:29,199 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd6a8fe798814cb296926c8374f2d80b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731309746298 2024-11-11T07:22:29,200 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1f5b47f0a4416e97ec7af9ee7df487, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731309747480 2024-11-11T07:22:29,200 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 720bc1f033314766b66b25b6346e3104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731309747480 2024-11-11T07:22:29,200 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ee901abe61f4eb192bd7bb482097ba5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731309747615 2024-11-11T07:22:29,200 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9a35f00f02423781d75ec0767c80b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731309747615 2024-11-11T07:22:29,210 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:29,211 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a49fb097a6a24592bc69ea3f08a90169 is 50, key is test_row_0/B:col10/1731309747616/Put/seqid=0 2024-11-11T07:22:29,215 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:29,215 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/3f70a21d62e4495e9f55564a230bf8d3 is 50, key is test_row_0/A:col10/1731309747616/Put/seqid=0 2024-11-11T07:22:29,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741918_1094 (size=13221) 2024-11-11T07:22:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741919_1095 (size=13221) 2024-11-11T07:22:29,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:22:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,341 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:22:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58 is 50, key is test_row_0/A:col10/1731309748287/Put/seqid=0 2024-11-11T07:22:29,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741920_1096 (size=12301) 2024-11-11T07:22:29,359 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58 2024-11-11T07:22:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/390915f0ecb24ddbb93a1521ae8d996a is 50, key is test_row_0/B:col10/1731309748287/Put/seqid=0 2024-11-11T07:22:29,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741921_1097 (size=12301) 2024-11-11T07:22:29,379 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/390915f0ecb24ddbb93a1521ae8d996a 2024-11-11T07:22:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/3e4d223d07ea4c768916449656f1c2a7 is 50, key is test_row_0/C:col10/1731309748287/Put/seqid=0 2024-11-11T07:22:29,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741922_1098 (size=12301) 2024-11-11T07:22:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:29,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309809452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309809456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309809457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309809462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309809573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309809573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309809573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309809573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,636 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/a49fb097a6a24592bc69ea3f08a90169 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a49fb097a6a24592bc69ea3f08a90169 2024-11-11T07:22:29,644 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into a49fb097a6a24592bc69ea3f08a90169(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:29,644 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:29,644 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309749197; duration=0sec 2024-11-11T07:22:29,644 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:29,644 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:29,644 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:29,646 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:29,646 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:29,646 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,646 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/de3b3f94d0ab413884c008353844a068, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/303f5a32862244ce8a0b1d7a2a9e5b88, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c5e07633d2d44957b2a75bd99fcf40cf] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=36.8 K 2024-11-11T07:22:29,647 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting de3b3f94d0ab413884c008353844a068, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731309746298 2024-11-11T07:22:29,647 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 303f5a32862244ce8a0b1d7a2a9e5b88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731309747480 2024-11-11T07:22:29,648 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/3f70a21d62e4495e9f55564a230bf8d3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/3f70a21d62e4495e9f55564a230bf8d3 2024-11-11T07:22:29,648 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c5e07633d2d44957b2a75bd99fcf40cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731309747615 2024-11-11T07:22:29,655 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into 3f70a21d62e4495e9f55564a230bf8d3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:29,655 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:29,655 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309749197; duration=0sec 2024-11-11T07:22:29,655 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:29,655 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:29,660 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:29,661 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/bdb50ce3135e4410a92c6b4ae811372c is 50, key is test_row_0/C:col10/1731309747616/Put/seqid=0 2024-11-11T07:22:29,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741923_1099 (size=13221) 2024-11-11T07:22:29,688 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/bdb50ce3135e4410a92c6b4ae811372c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdb50ce3135e4410a92c6b4ae811372c 2024-11-11T07:22:29,702 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into bdb50ce3135e4410a92c6b4ae811372c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:29,702 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:29,702 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309749197; duration=0sec 2024-11-11T07:22:29,702 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:29,702 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309809781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309809781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309809781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309809782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:29,816 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/3e4d223d07ea4c768916449656f1c2a7 2024-11-11T07:22:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58 2024-11-11T07:22:29,841 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58, entries=150, sequenceid=391, filesize=12.0 K 2024-11-11T07:22:29,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/390915f0ecb24ddbb93a1521ae8d996a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/390915f0ecb24ddbb93a1521ae8d996a 2024-11-11T07:22:29,854 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/390915f0ecb24ddbb93a1521ae8d996a, entries=150, sequenceid=391, filesize=12.0 K 2024-11-11T07:22:29,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/3e4d223d07ea4c768916449656f1c2a7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/3e4d223d07ea4c768916449656f1c2a7 2024-11-11T07:22:29,866 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/3e4d223d07ea4c768916449656f1c2a7, entries=150, sequenceid=391, filesize=12.0 K 2024-11-11T07:22:29,868 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5a2be9895ea70c6a849961e6f35c48df in 528ms, sequenceid=391, compaction requested=false 2024-11-11T07:22:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:29,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:29,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T07:22:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T07:22:29,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T07:22:29,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-11-11T07:22:29,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1020 sec 2024-11-11T07:22:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:22:29,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:29,922 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T07:22:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:22:29,924 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:29,925 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:29,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:22:30,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:22:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:30,083 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:22:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/9472b719ac734791a34b2e83b47660a8 is 50, key is test_row_0/A:col10/1731309749449/Put/seqid=0 2024-11-11T07:22:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:30,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:30,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741924_1100 (size=12301) 2024-11-11T07:22:30,095 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/9472b719ac734791a34b2e83b47660a8 2024-11-11T07:22:30,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/20322b0be0ea4e8a8a9ea547ca63b9ed is 50, key is test_row_0/B:col10/1731309749449/Put/seqid=0 2024-11-11T07:22:30,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309810124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309810125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309810126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309810128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741925_1101 (size=12301) 2024-11-11T07:22:30,146 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/20322b0be0ea4e8a8a9ea547ca63b9ed 2024-11-11T07:22:30,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/7e7d7724057d48919e1a6fc9a1f9114c is 50, key is test_row_0/C:col10/1731309749449/Put/seqid=0 2024-11-11T07:22:30,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741926_1102 (size=12301) 2024-11-11T07:22:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309810231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309810231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309810232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:22:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309810242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309810441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309810442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309810442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309810452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:22:30,571 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/7e7d7724057d48919e1a6fc9a1f9114c 2024-11-11T07:22:30,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/9472b719ac734791a34b2e83b47660a8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/9472b719ac734791a34b2e83b47660a8 2024-11-11T07:22:30,586 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/9472b719ac734791a34b2e83b47660a8, entries=150, sequenceid=407, filesize=12.0 K 2024-11-11T07:22:30,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/20322b0be0ea4e8a8a9ea547ca63b9ed as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/20322b0be0ea4e8a8a9ea547ca63b9ed 2024-11-11T07:22:30,594 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/20322b0be0ea4e8a8a9ea547ca63b9ed, entries=150, sequenceid=407, filesize=12.0 K 2024-11-11T07:22:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/7e7d7724057d48919e1a6fc9a1f9114c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7e7d7724057d48919e1a6fc9a1f9114c 2024-11-11T07:22:30,601 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7e7d7724057d48919e1a6fc9a1f9114c, entries=150, sequenceid=407, filesize=12.0 K 2024-11-11T07:22:30,603 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5a2be9895ea70c6a849961e6f35c48df in 521ms, sequenceid=407, compaction requested=true 2024-11-11T07:22:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T07:22:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T07:22:30,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T07:22:30,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 680 msec 2024-11-11T07:22:30,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 686 msec 2024-11-11T07:22:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:30,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:30,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:22:30,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/57cc4afdd30249278ba0e78c025ff824 is 50, key is test_row_0/A:col10/1731309750124/Put/seqid=0 2024-11-11T07:22:30,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741927_1103 (size=14741) 2024-11-11T07:22:30,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309810767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309810808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309810808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309810809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309810911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309810921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309810922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309810922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:22:31,061 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:31,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T07:22:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:22:31,065 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:31,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:31,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:31,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309811122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309811132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309811132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309811132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:22:31,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/57cc4afdd30249278ba0e78c025ff824 2024-11-11T07:22:31,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2fb9689d565a41d88ef78b705e804127 is 50, key is test_row_0/B:col10/1731309750124/Put/seqid=0 2024-11-11T07:22:31,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741928_1104 (size=12301) 2024-11-11T07:22:31,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T07:22:31,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:31,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T07:22:31,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:31,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:22:31,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309811432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309811441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309811441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309811441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:31,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:31,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T07:22:31,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:31,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2fb9689d565a41d88ef78b705e804127 2024-11-11T07:22:31,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/69205063d25b43baa48f099ac90a353d is 50, key is test_row_0/C:col10/1731309750124/Put/seqid=0 2024-11-11T07:22:31,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741929_1105 (size=12301) 2024-11-11T07:22:31,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/69205063d25b43baa48f099ac90a353d 2024-11-11T07:22:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/57cc4afdd30249278ba0e78c025ff824 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/57cc4afdd30249278ba0e78c025ff824 2024-11-11T07:22:31,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/57cc4afdd30249278ba0e78c025ff824, entries=200, sequenceid=432, filesize=14.4 K 2024-11-11T07:22:31,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/2fb9689d565a41d88ef78b705e804127 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2fb9689d565a41d88ef78b705e804127 2024-11-11T07:22:31,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2fb9689d565a41d88ef78b705e804127, entries=150, sequenceid=432, filesize=12.0 K 2024-11-11T07:22:31,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/69205063d25b43baa48f099ac90a353d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/69205063d25b43baa48f099ac90a353d 2024-11-11T07:22:31,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/69205063d25b43baa48f099ac90a353d, entries=150, sequenceid=432, filesize=12.0 K 2024-11-11T07:22:31,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T07:22:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:31,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5a2be9895ea70c6a849961e6f35c48df in 925ms, sequenceid=432, compaction requested=true 2024-11-11T07:22:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:31,683 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:31,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:31,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:31,684 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:31,686 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:31,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:31,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:31,686 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:31,686 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,686 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,687 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/3f70a21d62e4495e9f55564a230bf8d3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/9472b719ac734791a34b2e83b47660a8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/57cc4afdd30249278ba0e78c025ff824] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=51.3 K 2024-11-11T07:22:31,687 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a49fb097a6a24592bc69ea3f08a90169, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/390915f0ecb24ddbb93a1521ae8d996a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/20322b0be0ea4e8a8a9ea547ca63b9ed, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2fb9689d565a41d88ef78b705e804127] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=48.9 K 2024-11-11T07:22:31,687 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f70a21d62e4495e9f55564a230bf8d3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731309747615 2024-11-11T07:22:31,687 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a49fb097a6a24592bc69ea3f08a90169, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731309747615 2024-11-11T07:22:31,688 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting faab4a6cb43c4bbd8a0d4c2ff43b0b58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731309748281 2024-11-11T07:22:31,688 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 390915f0ecb24ddbb93a1521ae8d996a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731309748281 2024-11-11T07:22:31,688 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9472b719ac734791a34b2e83b47660a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731309749449 2024-11-11T07:22:31,688 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 20322b0be0ea4e8a8a9ea547ca63b9ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731309749449 2024-11-11T07:22:31,689 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57cc4afdd30249278ba0e78c025ff824, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731309750115 2024-11-11T07:22:31,689 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb9689d565a41d88ef78b705e804127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731309750124 2024-11-11T07:22:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:22:31,707 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:31,708 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c55a9a084aba43e684237d00d7348314 is 50, key is test_row_0/A:col10/1731309750124/Put/seqid=0 2024-11-11T07:22:31,712 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:31,713 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/7bcfb8e5211a40fea909acdc3e35a394 is 50, key is test_row_0/B:col10/1731309750124/Put/seqid=0 2024-11-11T07:22:31,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741930_1106 (size=13357) 2024-11-11T07:22:31,737 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c55a9a084aba43e684237d00d7348314 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c55a9a084aba43e684237d00d7348314 2024-11-11T07:22:31,747 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into c55a9a084aba43e684237d00d7348314(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:31,747 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:31,747 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=12, startTime=1731309751683; duration=0sec 2024-11-11T07:22:31,747 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:31,748 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:31,748 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:31,750 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:31,750 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:31,750 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,750 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdb50ce3135e4410a92c6b4ae811372c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/3e4d223d07ea4c768916449656f1c2a7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7e7d7724057d48919e1a6fc9a1f9114c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/69205063d25b43baa48f099ac90a353d] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=48.9 K 2024-11-11T07:22:31,753 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb50ce3135e4410a92c6b4ae811372c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731309747615 2024-11-11T07:22:31,753 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4d223d07ea4c768916449656f1c2a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731309748281 2024-11-11T07:22:31,755 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e7d7724057d48919e1a6fc9a1f9114c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731309749449 2024-11-11T07:22:31,755 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69205063d25b43baa48f099ac90a353d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731309750124 2024-11-11T07:22:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741931_1107 (size=13357) 2024-11-11T07:22:31,787 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:31,788 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b70aa1d09ed046ec8e47dd320b91d3cc is 50, key is test_row_0/C:col10/1731309750124/Put/seqid=0 2024-11-11T07:22:31,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741932_1108 (size=13357) 2024-11-11T07:22:31,824 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/b70aa1d09ed046ec8e47dd320b91d3cc as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b70aa1d09ed046ec8e47dd320b91d3cc 2024-11-11T07:22:31,834 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into b70aa1d09ed046ec8e47dd320b91d3cc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:31,834 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:31,834 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=12, startTime=1731309751683; duration=0sec 2024-11-11T07:22:31,834 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:31,834 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:31,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T07:22:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:31,836 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:31,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/13c0dc5ba7f64717a15a7704095e901d is 50, key is test_row_0/A:col10/1731309750776/Put/seqid=0 2024-11-11T07:22:31,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741933_1109 (size=12301) 2024-11-11T07:22:31,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:31,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:32,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309812012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309812012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309812018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309812018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309812122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309812132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309812132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309812132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,172 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/7bcfb8e5211a40fea909acdc3e35a394 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7bcfb8e5211a40fea909acdc3e35a394 2024-11-11T07:22:32,181 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 7bcfb8e5211a40fea909acdc3e35a394(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:32,181 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:32,181 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=12, startTime=1731309751683; duration=0sec 2024-11-11T07:22:32,181 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:32,181 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:22:32,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/13c0dc5ba7f64717a15a7704095e901d 2024-11-11T07:22:32,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/e022c4d9a7704e1aa090d085efea1ebe is 50, key is test_row_0/B:col10/1731309750776/Put/seqid=0 2024-11-11T07:22:32,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741934_1110 (size=12301) 2024-11-11T07:22:32,276 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/e022c4d9a7704e1aa090d085efea1ebe 2024-11-11T07:22:32,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/5a21a12c25b84134bb890606f47464a7 is 50, key is test_row_0/C:col10/1731309750776/Put/seqid=0 2024-11-11T07:22:32,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741935_1111 (size=12301) 2024-11-11T07:22:32,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309812331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309812342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309812342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309812342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309812642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309812651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309812651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309812661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,697 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/5a21a12c25b84134bb890606f47464a7 2024-11-11T07:22:32,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/13c0dc5ba7f64717a15a7704095e901d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/13c0dc5ba7f64717a15a7704095e901d 2024-11-11T07:22:32,713 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/13c0dc5ba7f64717a15a7704095e901d, entries=150, sequenceid=445, filesize=12.0 K 2024-11-11T07:22:32,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/e022c4d9a7704e1aa090d085efea1ebe as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e022c4d9a7704e1aa090d085efea1ebe 2024-11-11T07:22:32,735 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e022c4d9a7704e1aa090d085efea1ebe, entries=150, sequenceid=445, filesize=12.0 K 2024-11-11T07:22:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/5a21a12c25b84134bb890606f47464a7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/5a21a12c25b84134bb890606f47464a7 2024-11-11T07:22:32,744 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/5a21a12c25b84134bb890606f47464a7, entries=150, sequenceid=445, filesize=12.0 K 2024-11-11T07:22:32,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 5a2be9895ea70c6a849961e6f35c48df in 910ms, sequenceid=445, compaction requested=false 2024-11-11T07:22:32,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:32,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:32,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T07:22:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T07:22:32,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T07:22:32,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6810 sec 2024-11-11T07:22:32,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6890 sec 2024-11-11T07:22:32,767 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:32,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T07:22:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:32,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c4f6bea021604892a363188fe55fd0aa is 50, key is test_row_0/A:col10/1731309752765/Put/seqid=0 2024-11-11T07:22:32,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309812800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:32,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741936_1112 (size=14741) 2024-11-11T07:22:32,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c4f6bea021604892a363188fe55fd0aa 2024-11-11T07:22:32,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/655f6d606f1347b4a938987a2e29620a is 50, key is test_row_0/B:col10/1731309752765/Put/seqid=0 2024-11-11T07:22:32,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741937_1113 (size=12301) 2024-11-11T07:22:32,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309812911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:33,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309813128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:33,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38858 deadline: 1731309813151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38860 deadline: 1731309813162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38850 deadline: 1731309813162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38842 deadline: 1731309813172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:22:33,212 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:33,213 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-11T07:22:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:22:33,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:33,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:33,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:33,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/655f6d606f1347b4a938987a2e29620a 2024-11-11T07:22:33,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/02aab7f5f2ab43809aa7590564c57070 is 50, key is test_row_0/C:col10/1731309752765/Put/seqid=0 2024-11-11T07:22:33,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741938_1114 (size=12301) 2024-11-11T07:22:33,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/02aab7f5f2ab43809aa7590564c57070 2024-11-11T07:22:33,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/c4f6bea021604892a363188fe55fd0aa as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c4f6bea021604892a363188fe55fd0aa 2024-11-11T07:22:33,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c4f6bea021604892a363188fe55fd0aa, entries=200, sequenceid=473, filesize=14.4 K 2024-11-11T07:22:33,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/655f6d606f1347b4a938987a2e29620a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/655f6d606f1347b4a938987a2e29620a 2024-11-11T07:22:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:22:33,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/655f6d606f1347b4a938987a2e29620a, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T07:22:33,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/02aab7f5f2ab43809aa7590564c57070 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/02aab7f5f2ab43809aa7590564c57070 2024-11-11T07:22:33,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/02aab7f5f2ab43809aa7590564c57070, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T07:22:33,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 5a2be9895ea70c6a849961e6f35c48df in 571ms, sequenceid=473, compaction requested=true 2024-11-11T07:22:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:33,338 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:22:33,339 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:33,339 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:33,339 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:33,340 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,340 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c55a9a084aba43e684237d00d7348314, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/13c0dc5ba7f64717a15a7704095e901d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c4f6bea021604892a363188fe55fd0aa] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=39.5 K 2024-11-11T07:22:33,340 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c55a9a084aba43e684237d00d7348314, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731309750124 2024-11-11T07:22:33,340 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:33,341 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:33,341 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,341 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7bcfb8e5211a40fea909acdc3e35a394, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e022c4d9a7704e1aa090d085efea1ebe, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/655f6d606f1347b4a938987a2e29620a] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=37.1 K 2024-11-11T07:22:33,341 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 13c0dc5ba7f64717a15a7704095e901d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731309750758 2024-11-11T07:22:33,341 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bcfb8e5211a40fea909acdc3e35a394, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731309750124 2024-11-11T07:22:33,341 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f6bea021604892a363188fe55fd0aa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731309752010 2024-11-11T07:22:33,342 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting e022c4d9a7704e1aa090d085efea1ebe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731309750758 2024-11-11T07:22:33,342 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655f6d606f1347b4a938987a2e29620a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731309752016 2024-11-11T07:22:33,359 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:33,360 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/168b188990ff4c9c996d6bebb95b29fe is 50, key is test_row_0/B:col10/1731309752765/Put/seqid=0 2024-11-11T07:22:33,363 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:33,363 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/24e3a12dff66427183b67606dee92dde is 50, key is test_row_0/A:col10/1731309752765/Put/seqid=0 2024-11-11T07:22:33,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T07:22:33,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,371 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T07:22:33,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/bbb5782a0348498cb289f7b6a62bfd20 is 50, key is test_row_0/A:col10/1731309752794/Put/seqid=0 2024-11-11T07:22:33,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741940_1116 (size=13459) 2024-11-11T07:22:33,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741939_1115 (size=13459) 2024-11-11T07:22:33,392 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/24e3a12dff66427183b67606dee92dde as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/24e3a12dff66427183b67606dee92dde 2024-11-11T07:22:33,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741941_1117 (size=9857) 2024-11-11T07:22:33,395 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/bbb5782a0348498cb289f7b6a62bfd20 2024-11-11T07:22:33,404 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/168b188990ff4c9c996d6bebb95b29fe as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/168b188990ff4c9c996d6bebb95b29fe 2024-11-11T07:22:33,405 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into 24e3a12dff66427183b67606dee92dde(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:33,405 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,405 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309753338; duration=0sec 2024-11-11T07:22:33,405 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:33,405 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:33,405 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:33,410 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:33,410 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:33,410 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,411 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b70aa1d09ed046ec8e47dd320b91d3cc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/5a21a12c25b84134bb890606f47464a7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/02aab7f5f2ab43809aa7590564c57070] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=37.1 K 2024-11-11T07:22:33,413 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b70aa1d09ed046ec8e47dd320b91d3cc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731309750124 2024-11-11T07:22:33,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/8a99982cd6d244c8a675e6b6555cd485 is 50, key is test_row_0/B:col10/1731309752794/Put/seqid=0 2024-11-11T07:22:33,414 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a21a12c25b84134bb890606f47464a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731309750758 2024-11-11T07:22:33,421 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 02aab7f5f2ab43809aa7590564c57070, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731309752016 2024-11-11T07:22:33,424 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into 168b188990ff4c9c996d6bebb95b29fe(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:33,424 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,424 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309753338; duration=0sec 2024-11-11T07:22:33,424 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:33,424 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:33,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741942_1118 (size=9857) 2024-11-11T07:22:33,432 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/8a99982cd6d244c8a675e6b6555cd485 2024-11-11T07:22:33,438 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:33,439 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/c1ceb43b35d24b4db97827dd481ee35a is 50, key is test_row_0/C:col10/1731309752765/Put/seqid=0 2024-11-11T07:22:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/6b9658f37b4d4cefbe65dc59bdd14d8b is 50, key is test_row_0/C:col10/1731309752794/Put/seqid=0 2024-11-11T07:22:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:33,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:33,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741944_1120 (size=9857) 2024-11-11T07:22:33,473 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/6b9658f37b4d4cefbe65dc59bdd14d8b 2024-11-11T07:22:33,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741943_1119 (size=13459) 2024-11-11T07:22:33,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/bbb5782a0348498cb289f7b6a62bfd20 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/bbb5782a0348498cb289f7b6a62bfd20 2024-11-11T07:22:33,486 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/c1ceb43b35d24b4db97827dd481ee35a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c1ceb43b35d24b4db97827dd481ee35a 2024-11-11T07:22:33,490 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/bbb5782a0348498cb289f7b6a62bfd20, entries=100, sequenceid=482, filesize=9.6 K 2024-11-11T07:22:33,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/8a99982cd6d244c8a675e6b6555cd485 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8a99982cd6d244c8a675e6b6555cd485 2024-11-11T07:22:33,494 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into c1ceb43b35d24b4db97827dd481ee35a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:33,494 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,494 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309753338; duration=0sec 2024-11-11T07:22:33,494 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:33,494 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:33,500 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8a99982cd6d244c8a675e6b6555cd485, entries=100, sequenceid=482, filesize=9.6 K 2024-11-11T07:22:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/6b9658f37b4d4cefbe65dc59bdd14d8b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/6b9658f37b4d4cefbe65dc59bdd14d8b 2024-11-11T07:22:33,509 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/6b9658f37b4d4cefbe65dc59bdd14d8b, entries=100, sequenceid=482, filesize=9.6 K 2024-11-11T07:22:33,510 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=67.09 KB/68700 for 5a2be9895ea70c6a849961e6f35c48df in 139ms, sequenceid=482, compaction requested=false 2024-11-11T07:22:33,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-11T07:22:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:33,511 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:33,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:22:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-11T07:22:33,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T07:22:33,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 296 msec 2024-11-11T07:22:33,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/65cdab3d43694da788b6c18a3cd58fdd is 50, key is test_row_0/A:col10/1731309753489/Put/seqid=0 2024-11-11T07:22:33,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 307 msec 2024-11-11T07:22:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:22:33,532 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:33,534 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-11T07:22:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:22:33,538 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:33,539 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:33,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741945_1121 (size=14741) 2024-11-11T07:22:33,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/65cdab3d43694da788b6c18a3cd58fdd 2024-11-11T07:22:33,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/061d064e11524da28e38443f9306b771 is 50, key is test_row_0/B:col10/1731309753489/Put/seqid=0 2024-11-11T07:22:33,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741946_1122 (size=12301) 2024-11-11T07:22:33,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/061d064e11524da28e38443f9306b771 2024-11-11T07:22:33,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/405b6425a882475080b3748e8aa2235e is 50, key is test_row_0/C:col10/1731309753489/Put/seqid=0 2024-11-11T07:22:33,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741947_1123 (size=12301) 2024-11-11T07:22:33,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/405b6425a882475080b3748e8aa2235e 2024-11-11T07:22:33,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/65cdab3d43694da788b6c18a3cd58fdd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/65cdab3d43694da788b6c18a3cd58fdd 2024-11-11T07:22:33,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/65cdab3d43694da788b6c18a3cd58fdd, entries=200, sequenceid=499, filesize=14.4 K 2024-11-11T07:22:33,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/061d064e11524da28e38443f9306b771 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/061d064e11524da28e38443f9306b771 2024-11-11T07:22:33,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/061d064e11524da28e38443f9306b771, entries=150, sequenceid=499, filesize=12.0 K 2024-11-11T07:22:33,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38890 deadline: 1731309813637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/405b6425a882475080b3748e8aa2235e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/405b6425a882475080b3748e8aa2235e 2024-11-11T07:22:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:22:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:22:33,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/405b6425a882475080b3748e8aa2235e, entries=150, sequenceid=499, filesize=12.0 K 2024-11-11T07:22:33,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5a2be9895ea70c6a849961e6f35c48df in 135ms, sequenceid=499, compaction requested=true 2024-11-11T07:22:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:33,647 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:33,647 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:33,649 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:33,649 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/B is initiating minor compaction (all files) 2024-11-11T07:22:33,649 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/B in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,649 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/168b188990ff4c9c996d6bebb95b29fe, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8a99982cd6d244c8a675e6b6555cd485, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/061d064e11524da28e38443f9306b771] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=34.8 K 2024-11-11T07:22:33,649 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:33,649 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/A is initiating minor compaction (all files) 2024-11-11T07:22:33,649 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/A in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,649 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 168b188990ff4c9c996d6bebb95b29fe, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731309752016 2024-11-11T07:22:33,650 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/24e3a12dff66427183b67606dee92dde, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/bbb5782a0348498cb289f7b6a62bfd20, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/65cdab3d43694da788b6c18a3cd58fdd] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=37.2 K 2024-11-11T07:22:33,650 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a99982cd6d244c8a675e6b6555cd485, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731309752794 2024-11-11T07:22:33,650 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e3a12dff66427183b67606dee92dde, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731309752016 2024-11-11T07:22:33,650 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 061d064e11524da28e38443f9306b771, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731309753489 2024-11-11T07:22:33,651 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb5782a0348498cb289f7b6a62bfd20, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731309752794 2024-11-11T07:22:33,651 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65cdab3d43694da788b6c18a3cd58fdd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731309753484 2024-11-11T07:22:33,670 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:33,671 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/d7d74dd9fbad4e50a101bb89c5601cc9 is 50, key is test_row_0/B:col10/1731309753489/Put/seqid=0 2024-11-11T07:22:33,674 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:33,675 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/af63a87a94544e38952b47c881a22132 is 50, key is test_row_0/A:col10/1731309753489/Put/seqid=0 2024-11-11T07:22:33,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741949_1125 (size=13561) 2024-11-11T07:22:33,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T07:22:33,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,694 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T07:22:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/09439a099ed241189e462925a2a56ace is 50, key is test_row_0/A:col10/1731309753604/Put/seqid=0 2024-11-11T07:22:33,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741948_1124 (size=13561) 2024-11-11T07:22:33,715 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/af63a87a94544e38952b47c881a22132 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/af63a87a94544e38952b47c881a22132 2024-11-11T07:22:33,724 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/A of 5a2be9895ea70c6a849961e6f35c48df into af63a87a94544e38952b47c881a22132(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:33,724 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,724 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/A, priority=13, startTime=1731309753647; duration=0sec 2024-11-11T07:22:33,724 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:33,724 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:33,724 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:33,726 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:33,726 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 5a2be9895ea70c6a849961e6f35c48df/C is initiating minor compaction (all files) 2024-11-11T07:22:33,726 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a2be9895ea70c6a849961e6f35c48df/C in TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,726 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c1ceb43b35d24b4db97827dd481ee35a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/6b9658f37b4d4cefbe65dc59bdd14d8b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/405b6425a882475080b3748e8aa2235e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp, totalSize=34.8 K 2024-11-11T07:22:33,728 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ceb43b35d24b4db97827dd481ee35a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731309752016 2024-11-11T07:22:33,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b9658f37b4d4cefbe65dc59bdd14d8b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731309752794 2024-11-11T07:22:33,731 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405b6425a882475080b3748e8aa2235e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731309753489 2024-11-11T07:22:33,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741950_1126 (size=12301) 2024-11-11T07:22:33,746 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/09439a099ed241189e462925a2a56ace 2024-11-11T07:22:33,758 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a2be9895ea70c6a849961e6f35c48df#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:33,759 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/c2cdb360772c4f1ba950d84668c167b8 is 50, key is test_row_0/C:col10/1731309753489/Put/seqid=0 2024-11-11T07:22:33,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/19345572e52d4fd3bfa06ed47dc73506 is 50, key is test_row_0/B:col10/1731309753604/Put/seqid=0 2024-11-11T07:22:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:33,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. as already flushing 2024-11-11T07:22:33,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741951_1127 (size=13561) 2024-11-11T07:22:33,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741952_1128 (size=12301) 2024-11-11T07:22:33,781 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/19345572e52d4fd3bfa06ed47dc73506 2024-11-11T07:22:33,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/bdf11220d3574b788741b4b871ff3bfa is 50, key is test_row_0/C:col10/1731309753604/Put/seqid=0 2024-11-11T07:22:33,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741953_1129 (size=12301) 2024-11-11T07:22:33,801 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/bdf11220d3574b788741b4b871ff3bfa 2024-11-11T07:22:33,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/09439a099ed241189e462925a2a56ace as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/09439a099ed241189e462925a2a56ace 2024-11-11T07:22:33,815 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/09439a099ed241189e462925a2a56ace, entries=150, sequenceid=521, filesize=12.0 K 2024-11-11T07:22:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/19345572e52d4fd3bfa06ed47dc73506 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/19345572e52d4fd3bfa06ed47dc73506 2024-11-11T07:22:33,826 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T07:22:33,827 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/19345572e52d4fd3bfa06ed47dc73506, entries=150, sequenceid=521, filesize=12.0 K 2024-11-11T07:22:33,827 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:33,828 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T07:22:33,828 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:33,828 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T07:22:33,828 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T07:22:33,828 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:33,828 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:33,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/bdf11220d3574b788741b4b871ff3bfa as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdf11220d3574b788741b4b871ff3bfa 2024-11-11T07:22:33,832 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T07:22:33,832 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:33,834 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,834 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,834 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,834 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,834 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdf11220d3574b788741b4b871ff3bfa, entries=150, sequenceid=521, filesize=12.0 K 2024-11-11T07:22:33,835 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,835 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:33,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:33,835 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,835 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:33,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:33,836 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,836 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:33,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:33,837 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=67.09 KB/68700 for 5a2be9895ea70c6a849961e6f35c48df in 143ms, sequenceid=521, compaction requested=false 2024-11-11T07:22:33,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:33,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:33,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-11T07:22:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-11T07:22:33,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T07:22:33,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 299 msec 2024-11-11T07:22:33,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 307 msec 2024-11-11T07:22:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:22:33,852 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:34,099 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/d7d74dd9fbad4e50a101bb89c5601cc9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/d7d74dd9fbad4e50a101bb89c5601cc9 2024-11-11T07:22:34,105 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/B of 5a2be9895ea70c6a849961e6f35c48df into d7d74dd9fbad4e50a101bb89c5601cc9(size=13.2 K), total size for store is 25.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:34,106 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:34,106 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/B, priority=13, startTime=1731309753647; duration=0sec 2024-11-11T07:22:34,106 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:34,106 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:34,163 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:34,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:22:34,163 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T07:22:34,163 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:34,164 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,164 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:34,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/69362cc70aef499ab4dcd8a461bfac37 is 50, key is test_row_0/A:col10/1731309753812/Put/seqid=0 2024-11-11T07:22:34,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741954_1130 (size=12301) 2024-11-11T07:22:34,174 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T07:22:34,174 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:34,174 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,174 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:34,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/69362cc70aef499ab4dcd8a461bfac37 2024-11-11T07:22:34,178 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/c2cdb360772c4f1ba950d84668c167b8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c2cdb360772c4f1ba950d84668c167b8 2024-11-11T07:22:34,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/4a690b086f644b248b48cab27feb035e is 50, key is test_row_0/B:col10/1731309753812/Put/seqid=0 2024-11-11T07:22:34,184 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T07:22:34,184 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:34,184 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a2be9895ea70c6a849961e6f35c48df/C of 5a2be9895ea70c6a849961e6f35c48df into c2cdb360772c4f1ba950d84668c167b8(size=13.2 K), total size for store is 25.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:34,184 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,184 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:34,184 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., storeName=5a2be9895ea70c6a849961e6f35c48df/C, priority=13, startTime=1731309753647; duration=0sec 2024-11-11T07:22:34,184 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:34,184 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:34,184 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:34,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741955_1131 (size=12301) 2024-11-11T07:22:34,192 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T07:22:34,192 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:22:34,192 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,193 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2994 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3064 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1390 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4165 rows 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1389 2024-11-11T07:22:34,193 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4166 rows 2024-11-11T07:22:34,194 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:22:34,194 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:22:34,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:34,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:22:34,199 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:22:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T07:22:34,207 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309754207"}]},"ts":"1731309754207"} 2024-11-11T07:22:34,209 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:22:34,209 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:22:34,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:22:34,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, UNASSIGN}] 2024-11-11T07:22:34,215 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, UNASSIGN 2024-11-11T07:22:34,216 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=5a2be9895ea70c6a849961e6f35c48df, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:34,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, UNASSIGN because future has completed 2024-11-11T07:22:34,218 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:22:34,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:22:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T07:22:34,376 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:34,376 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:22:34,377 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 5a2be9895ea70c6a849961e6f35c48df, disabling compactions & flushes 2024-11-11T07:22:34,377 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T07:22:34,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/4a690b086f644b248b48cab27feb035e 2024-11-11T07:22:34,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/a8617a72fc55404fab1d525094cea823 is 50, key is test_row_0/C:col10/1731309753812/Put/seqid=0 2024-11-11T07:22:34,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741956_1132 (size=12301) 2024-11-11T07:22:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T07:22:35,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/a8617a72fc55404fab1d525094cea823 2024-11-11T07:22:35,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/69362cc70aef499ab4dcd8a461bfac37 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/69362cc70aef499ab4dcd8a461bfac37 2024-11-11T07:22:35,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/69362cc70aef499ab4dcd8a461bfac37, entries=150, sequenceid=537, filesize=12.0 K 2024-11-11T07:22:35,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/4a690b086f644b248b48cab27feb035e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/4a690b086f644b248b48cab27feb035e 2024-11-11T07:22:35,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/4a690b086f644b248b48cab27feb035e, entries=150, sequenceid=537, filesize=12.0 K 2024-11-11T07:22:35,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/a8617a72fc55404fab1d525094cea823 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a8617a72fc55404fab1d525094cea823 2024-11-11T07:22:35,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a8617a72fc55404fab1d525094cea823, entries=150, sequenceid=537, filesize=12.0 K 2024-11-11T07:22:35,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=20.13 KB/20610 for 5a2be9895ea70c6a849961e6f35c48df in 858ms, sequenceid=537, compaction requested=true 2024-11-11T07:22:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a2be9895ea70c6a849961e6f35c48df: 2024-11-11T07:22:35,021 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:35,021 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:35,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:35,022 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. after waiting 0 ms 2024-11-11T07:22:35,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:35,022 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:35,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a2be9895ea70c6a849961e6f35c48df:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:35,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:35,022 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. because compaction request was cancelled 2024-11-11T07:22:35,022 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. because compaction request was cancelled 2024-11-11T07:22:35,022 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:B 2024-11-11T07:22:35,022 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:A 2024-11-11T07:22:35,022 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 5a2be9895ea70c6a849961e6f35c48df 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-11T07:22:35,022 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. because compaction request was cancelled 2024-11-11T07:22:35,022 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a2be9895ea70c6a849961e6f35c48df:C 2024-11-11T07:22:35,026 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/176e582021954d35b843b2fcda073321 is 50, key is test_row_0/A:col10/1731309754191/Put/seqid=0 2024-11-11T07:22:35,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741957_1133 (size=9857) 2024-11-11T07:22:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T07:22:35,433 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/176e582021954d35b843b2fcda073321 2024-11-11T07:22:35,441 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/49c37ab4571945058104b3193e88d357 is 50, key is test_row_0/B:col10/1731309754191/Put/seqid=0 2024-11-11T07:22:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741958_1134 (size=9857) 2024-11-11T07:22:35,846 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/49c37ab4571945058104b3193e88d357 2024-11-11T07:22:35,854 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/1505877823ed46ceab3be7926f907b56 is 50, key is test_row_0/C:col10/1731309754191/Put/seqid=0 2024-11-11T07:22:35,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741959_1135 (size=9857) 2024-11-11T07:22:36,258 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/1505877823ed46ceab3be7926f907b56 2024-11-11T07:22:36,264 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/A/176e582021954d35b843b2fcda073321 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/176e582021954d35b843b2fcda073321 2024-11-11T07:22:36,268 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/176e582021954d35b843b2fcda073321, entries=100, sequenceid=544, filesize=9.6 K 2024-11-11T07:22:36,269 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/B/49c37ab4571945058104b3193e88d357 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49c37ab4571945058104b3193e88d357 2024-11-11T07:22:36,273 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49c37ab4571945058104b3193e88d357, entries=100, sequenceid=544, filesize=9.6 K 2024-11-11T07:22:36,274 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/.tmp/C/1505877823ed46ceab3be7926f907b56 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/1505877823ed46ceab3be7926f907b56 2024-11-11T07:22:36,278 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/1505877823ed46ceab3be7926f907b56, entries=100, sequenceid=544, filesize=9.6 K 2024-11-11T07:22:36,279 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 5a2be9895ea70c6a849961e6f35c48df in 1257ms, sequenceid=544, compaction requested=true 2024-11-11T07:22:36,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/1affbadaee5c4ef88d916c59e17ce2af, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b668c16d28534e52be6e13f77d4aa0fc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c3e9f675087647a4a8fd1044547aeb18, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c49a4ee40e5b48aa8539e8772cd66fa0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/892187065a854ff2b817ea9ab448a69d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a1b2ddeaf18e4dcab4035f4919a9db82, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b02e3f0638404aca842d1b6df54978e6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a5b30088eb7641c8adf249ed715cbe91, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/cb655b115c0a42a789b52f00382b5867, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/956d865d94dd48f7b5806e835203049b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/63eaa9cd24dd449396873481277a1042, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/06d4690251dc4d51ae873e2e4d4564ea, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/7f13415c755c4b11bba9e6e85d55bd34, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d8401429138548449ea7335851d47095, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/6f2a43cccfb14801b9f1fd8f8f27e312, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/67ac483929f4472f8ed9a270849cdd8d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d724d662dade4e8887a56c53babe1bee, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/58cd5c9a387744d688b19dad70907a4c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ec05240400d44ae3a2daff9fd049f84a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b75bcf69aa32498bb03f221e1d417e85, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/96b019ed72064473b25c634f20ce76a2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/eddfed098f5c45d797a54789e85efc3b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ddc33e3475984078ae37ff06247962b9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/fd6a8fe798814cb296926c8374f2d80b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/00fcd316daf146d3b2a874a8da567ee1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ba1f5b47f0a4416e97ec7af9ee7df487, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/8ee901abe61f4eb192bd7bb482097ba5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/3f70a21d62e4495e9f55564a230bf8d3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/9472b719ac734791a34b2e83b47660a8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/57cc4afdd30249278ba0e78c025ff824, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c55a9a084aba43e684237d00d7348314, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/13c0dc5ba7f64717a15a7704095e901d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c4f6bea021604892a363188fe55fd0aa, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/24e3a12dff66427183b67606dee92dde, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/bbb5782a0348498cb289f7b6a62bfd20, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/65cdab3d43694da788b6c18a3cd58fdd] to archive 2024-11-11T07:22:36,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:22:36,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/1affbadaee5c4ef88d916c59e17ce2af to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/1affbadaee5c4ef88d916c59e17ce2af 2024-11-11T07:22:36,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b668c16d28534e52be6e13f77d4aa0fc to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b668c16d28534e52be6e13f77d4aa0fc 2024-11-11T07:22:36,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c3e9f675087647a4a8fd1044547aeb18 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c3e9f675087647a4a8fd1044547aeb18 2024-11-11T07:22:36,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c49a4ee40e5b48aa8539e8772cd66fa0 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c49a4ee40e5b48aa8539e8772cd66fa0 2024-11-11T07:22:36,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/892187065a854ff2b817ea9ab448a69d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/892187065a854ff2b817ea9ab448a69d 2024-11-11T07:22:36,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a1b2ddeaf18e4dcab4035f4919a9db82 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a1b2ddeaf18e4dcab4035f4919a9db82 2024-11-11T07:22:36,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b02e3f0638404aca842d1b6df54978e6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b02e3f0638404aca842d1b6df54978e6 2024-11-11T07:22:36,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a5b30088eb7641c8adf249ed715cbe91 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/a5b30088eb7641c8adf249ed715cbe91 2024-11-11T07:22:36,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/cb655b115c0a42a789b52f00382b5867 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/cb655b115c0a42a789b52f00382b5867 2024-11-11T07:22:36,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/956d865d94dd48f7b5806e835203049b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/956d865d94dd48f7b5806e835203049b 2024-11-11T07:22:36,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/63eaa9cd24dd449396873481277a1042 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/63eaa9cd24dd449396873481277a1042 2024-11-11T07:22:36,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/06d4690251dc4d51ae873e2e4d4564ea to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/06d4690251dc4d51ae873e2e4d4564ea 2024-11-11T07:22:36,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/7f13415c755c4b11bba9e6e85d55bd34 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/7f13415c755c4b11bba9e6e85d55bd34 2024-11-11T07:22:36,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d8401429138548449ea7335851d47095 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d8401429138548449ea7335851d47095 2024-11-11T07:22:36,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/6f2a43cccfb14801b9f1fd8f8f27e312 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/6f2a43cccfb14801b9f1fd8f8f27e312 2024-11-11T07:22:36,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/67ac483929f4472f8ed9a270849cdd8d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/67ac483929f4472f8ed9a270849cdd8d 2024-11-11T07:22:36,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d724d662dade4e8887a56c53babe1bee to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/d724d662dade4e8887a56c53babe1bee 2024-11-11T07:22:36,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/58cd5c9a387744d688b19dad70907a4c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/58cd5c9a387744d688b19dad70907a4c 2024-11-11T07:22:36,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ec05240400d44ae3a2daff9fd049f84a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ec05240400d44ae3a2daff9fd049f84a 2024-11-11T07:22:36,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b75bcf69aa32498bb03f221e1d417e85 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/b75bcf69aa32498bb03f221e1d417e85 2024-11-11T07:22:36,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/96b019ed72064473b25c634f20ce76a2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/96b019ed72064473b25c634f20ce76a2 2024-11-11T07:22:36,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/eddfed098f5c45d797a54789e85efc3b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/eddfed098f5c45d797a54789e85efc3b 2024-11-11T07:22:36,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ddc33e3475984078ae37ff06247962b9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ddc33e3475984078ae37ff06247962b9 2024-11-11T07:22:36,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/fd6a8fe798814cb296926c8374f2d80b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/fd6a8fe798814cb296926c8374f2d80b 2024-11-11T07:22:36,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/00fcd316daf146d3b2a874a8da567ee1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/00fcd316daf146d3b2a874a8da567ee1 2024-11-11T07:22:36,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ba1f5b47f0a4416e97ec7af9ee7df487 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/ba1f5b47f0a4416e97ec7af9ee7df487 2024-11-11T07:22:36,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/8ee901abe61f4eb192bd7bb482097ba5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/8ee901abe61f4eb192bd7bb482097ba5 2024-11-11T07:22:36,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/3f70a21d62e4495e9f55564a230bf8d3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/3f70a21d62e4495e9f55564a230bf8d3 2024-11-11T07:22:36,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/faab4a6cb43c4bbd8a0d4c2ff43b0b58 2024-11-11T07:22:36,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/9472b719ac734791a34b2e83b47660a8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/9472b719ac734791a34b2e83b47660a8 2024-11-11T07:22:36,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/57cc4afdd30249278ba0e78c025ff824 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/57cc4afdd30249278ba0e78c025ff824 2024-11-11T07:22:36,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c55a9a084aba43e684237d00d7348314 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c55a9a084aba43e684237d00d7348314 2024-11-11T07:22:36,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/13c0dc5ba7f64717a15a7704095e901d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/13c0dc5ba7f64717a15a7704095e901d 2024-11-11T07:22:36,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c4f6bea021604892a363188fe55fd0aa to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/c4f6bea021604892a363188fe55fd0aa 2024-11-11T07:22:36,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/24e3a12dff66427183b67606dee92dde to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/24e3a12dff66427183b67606dee92dde 2024-11-11T07:22:36,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/bbb5782a0348498cb289f7b6a62bfd20 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/bbb5782a0348498cb289f7b6a62bfd20 2024-11-11T07:22:36,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/65cdab3d43694da788b6c18a3cd58fdd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/65cdab3d43694da788b6c18a3cd58fdd 2024-11-11T07:22:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T07:22:36,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/dc05cca9620f46daab7b1218d338a10d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/bfb4d4af6ec5446b90f6679d56451c8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0621882202ee4ce28b4b0751aa2b753f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a6060d115317489b9affc3a2ef9d39fd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a4892ed7ab1f4216b540840b08a2dcbd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/756308c1e3d94ec3b177047c36d241d5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/3d56e8bf6a0c460fa22b72d9b3f8619e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49338511c2b44869a9a5c9ce500d059d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/9433bc4b23bb423d8668a9db0cdc6bdf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/29248ab0cf6b4a95b906f0f495dd6c3f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/592da6ddfe3842228ddfa527fc89ec01, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/82a342918b964630be52b932e94f2e59, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/f7cde8959a7442daa876e7f321a49290, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6aa12f1058b449b79a161b5c04cd104b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8815ccbe49da49789aa2e5a3a8c56766, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7a068c05ba2b437e9e52340a709c5489, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1767e4af16ea482ca85825725422f795, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6d5ab97ae92f425fb217b2aed7872855, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/50682463056547b5a8883aeab461eedc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1371aa06186b4261b24cc48a38be6531, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0e5490400e9d48cf8d2bf7694b3d11d1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2671476d274544899ee102aefc00f5cb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/033a20ca01434563b191ceeff58a6a44, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e2bb6957cb7e4efbae4d2f6aa6bd6307, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/737c2e5c55a5496c91d35fdbd2e99abf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/720bc1f033314766b66b25b6346e3104, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a49fb097a6a24592bc69ea3f08a90169, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2c9a35f00f02423781d75ec0767c80b8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/390915f0ecb24ddbb93a1521ae8d996a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/20322b0be0ea4e8a8a9ea547ca63b9ed, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7bcfb8e5211a40fea909acdc3e35a394, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2fb9689d565a41d88ef78b705e804127, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e022c4d9a7704e1aa090d085efea1ebe, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/168b188990ff4c9c996d6bebb95b29fe, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/655f6d606f1347b4a938987a2e29620a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8a99982cd6d244c8a675e6b6555cd485, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/061d064e11524da28e38443f9306b771] to archive 2024-11-11T07:22:36,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:22:36,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/dc05cca9620f46daab7b1218d338a10d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/dc05cca9620f46daab7b1218d338a10d 2024-11-11T07:22:36,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/bfb4d4af6ec5446b90f6679d56451c8c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/bfb4d4af6ec5446b90f6679d56451c8c 2024-11-11T07:22:36,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0621882202ee4ce28b4b0751aa2b753f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0621882202ee4ce28b4b0751aa2b753f 2024-11-11T07:22:36,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a6060d115317489b9affc3a2ef9d39fd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a6060d115317489b9affc3a2ef9d39fd 2024-11-11T07:22:36,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a4892ed7ab1f4216b540840b08a2dcbd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a4892ed7ab1f4216b540840b08a2dcbd 2024-11-11T07:22:36,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/756308c1e3d94ec3b177047c36d241d5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/756308c1e3d94ec3b177047c36d241d5 2024-11-11T07:22:36,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/3d56e8bf6a0c460fa22b72d9b3f8619e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/3d56e8bf6a0c460fa22b72d9b3f8619e 2024-11-11T07:22:36,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49338511c2b44869a9a5c9ce500d059d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49338511c2b44869a9a5c9ce500d059d 2024-11-11T07:22:36,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/9433bc4b23bb423d8668a9db0cdc6bdf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/9433bc4b23bb423d8668a9db0cdc6bdf 2024-11-11T07:22:36,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/29248ab0cf6b4a95b906f0f495dd6c3f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/29248ab0cf6b4a95b906f0f495dd6c3f 2024-11-11T07:22:36,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/592da6ddfe3842228ddfa527fc89ec01 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/592da6ddfe3842228ddfa527fc89ec01 2024-11-11T07:22:36,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/82a342918b964630be52b932e94f2e59 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/82a342918b964630be52b932e94f2e59 2024-11-11T07:22:36,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/f7cde8959a7442daa876e7f321a49290 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/f7cde8959a7442daa876e7f321a49290 2024-11-11T07:22:36,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6aa12f1058b449b79a161b5c04cd104b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6aa12f1058b449b79a161b5c04cd104b 2024-11-11T07:22:36,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8815ccbe49da49789aa2e5a3a8c56766 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8815ccbe49da49789aa2e5a3a8c56766 2024-11-11T07:22:36,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7a068c05ba2b437e9e52340a709c5489 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7a068c05ba2b437e9e52340a709c5489 2024-11-11T07:22:36,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1767e4af16ea482ca85825725422f795 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1767e4af16ea482ca85825725422f795 2024-11-11T07:22:36,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6d5ab97ae92f425fb217b2aed7872855 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/6d5ab97ae92f425fb217b2aed7872855 2024-11-11T07:22:36,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/50682463056547b5a8883aeab461eedc to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/50682463056547b5a8883aeab461eedc 2024-11-11T07:22:36,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1371aa06186b4261b24cc48a38be6531 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/1371aa06186b4261b24cc48a38be6531 2024-11-11T07:22:36,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0e5490400e9d48cf8d2bf7694b3d11d1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/0e5490400e9d48cf8d2bf7694b3d11d1 2024-11-11T07:22:36,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2671476d274544899ee102aefc00f5cb to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2671476d274544899ee102aefc00f5cb 2024-11-11T07:22:36,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/033a20ca01434563b191ceeff58a6a44 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/033a20ca01434563b191ceeff58a6a44 2024-11-11T07:22:36,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e2bb6957cb7e4efbae4d2f6aa6bd6307 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e2bb6957cb7e4efbae4d2f6aa6bd6307 2024-11-11T07:22:36,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/737c2e5c55a5496c91d35fdbd2e99abf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/737c2e5c55a5496c91d35fdbd2e99abf 2024-11-11T07:22:36,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/720bc1f033314766b66b25b6346e3104 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/720bc1f033314766b66b25b6346e3104 2024-11-11T07:22:36,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a49fb097a6a24592bc69ea3f08a90169 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/a49fb097a6a24592bc69ea3f08a90169 2024-11-11T07:22:36,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2c9a35f00f02423781d75ec0767c80b8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2c9a35f00f02423781d75ec0767c80b8 2024-11-11T07:22:36,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/390915f0ecb24ddbb93a1521ae8d996a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/390915f0ecb24ddbb93a1521ae8d996a 2024-11-11T07:22:36,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/20322b0be0ea4e8a8a9ea547ca63b9ed to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/20322b0be0ea4e8a8a9ea547ca63b9ed 2024-11-11T07:22:36,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7bcfb8e5211a40fea909acdc3e35a394 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/7bcfb8e5211a40fea909acdc3e35a394 2024-11-11T07:22:36,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2fb9689d565a41d88ef78b705e804127 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/2fb9689d565a41d88ef78b705e804127 2024-11-11T07:22:36,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e022c4d9a7704e1aa090d085efea1ebe to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/e022c4d9a7704e1aa090d085efea1ebe 2024-11-11T07:22:36,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/168b188990ff4c9c996d6bebb95b29fe to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/168b188990ff4c9c996d6bebb95b29fe 2024-11-11T07:22:36,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/655f6d606f1347b4a938987a2e29620a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/655f6d606f1347b4a938987a2e29620a 2024-11-11T07:22:36,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8a99982cd6d244c8a675e6b6555cd485 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/8a99982cd6d244c8a675e6b6555cd485 2024-11-11T07:22:36,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/061d064e11524da28e38443f9306b771 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/061d064e11524da28e38443f9306b771 2024-11-11T07:22:36,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/14b9492525e047789437254c35962bc4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/35dcc4f4373a44f0b8c6350f4bca69bb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7bc017ba53a341508780bd8c5fcfe2ff, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b7292c4fa3754ae28f0ecea07dc00363, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b0398632fbf5439fad7fd7218269b70f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4bae24d2815248dcba8f84269d19cf76, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/91b2e74cc3464671bdfdf808358d88d9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/310e90ea77b54f4e9df8a9179ec63bac, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9d72aa7540b847829e9f55136662021f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/44cd84be721a40e6b4ca1bd3202b6d66, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a14781530ba24ff7b3becba907d663b5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/cea3ab140ff64f8790db97b5d46e8691, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f4395c9376c04496b3c532b15239f4ac, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/325f622680934044bf477167c368b1f6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/abf9329f1b8c4010bc87ed676fc50248, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/72863b2b5c7b4439abe5dcee6a935b3c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f0b3db212fa74ca590ac54e7c6acf8a9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f85be4ef0e21433083daa79ccf7f37e6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f98d8303b8864b2cb25cc8834fbc630f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b14724340dc04cd8a632a128007cc8b9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4fbf2ac53d7a4f209d47d25ddc379772, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9943559c00cb462486f969082100d955, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4eb3b07feef74466839135a03caf4398, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/de3b3f94d0ab413884c008353844a068, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/13995f07c4d6474e9f541dddab5ba6f4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/303f5a32862244ce8a0b1d7a2a9e5b88, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdb50ce3135e4410a92c6b4ae811372c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c5e07633d2d44957b2a75bd99fcf40cf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/3e4d223d07ea4c768916449656f1c2a7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7e7d7724057d48919e1a6fc9a1f9114c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b70aa1d09ed046ec8e47dd320b91d3cc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/69205063d25b43baa48f099ac90a353d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/5a21a12c25b84134bb890606f47464a7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c1ceb43b35d24b4db97827dd481ee35a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/02aab7f5f2ab43809aa7590564c57070, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/6b9658f37b4d4cefbe65dc59bdd14d8b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/405b6425a882475080b3748e8aa2235e] to archive 2024-11-11T07:22:36,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:22:36,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/14b9492525e047789437254c35962bc4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/14b9492525e047789437254c35962bc4 2024-11-11T07:22:36,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/35dcc4f4373a44f0b8c6350f4bca69bb to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/35dcc4f4373a44f0b8c6350f4bca69bb 2024-11-11T07:22:36,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7bc017ba53a341508780bd8c5fcfe2ff to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7bc017ba53a341508780bd8c5fcfe2ff 2024-11-11T07:22:36,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b7292c4fa3754ae28f0ecea07dc00363 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b7292c4fa3754ae28f0ecea07dc00363 2024-11-11T07:22:36,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b0398632fbf5439fad7fd7218269b70f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b0398632fbf5439fad7fd7218269b70f 2024-11-11T07:22:36,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4bae24d2815248dcba8f84269d19cf76 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4bae24d2815248dcba8f84269d19cf76 2024-11-11T07:22:36,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/91b2e74cc3464671bdfdf808358d88d9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/91b2e74cc3464671bdfdf808358d88d9 2024-11-11T07:22:36,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/310e90ea77b54f4e9df8a9179ec63bac to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/310e90ea77b54f4e9df8a9179ec63bac 2024-11-11T07:22:36,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9d72aa7540b847829e9f55136662021f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9d72aa7540b847829e9f55136662021f 2024-11-11T07:22:36,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/44cd84be721a40e6b4ca1bd3202b6d66 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/44cd84be721a40e6b4ca1bd3202b6d66 2024-11-11T07:22:36,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a14781530ba24ff7b3becba907d663b5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a14781530ba24ff7b3becba907d663b5 2024-11-11T07:22:36,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/cea3ab140ff64f8790db97b5d46e8691 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/cea3ab140ff64f8790db97b5d46e8691 2024-11-11T07:22:36,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f4395c9376c04496b3c532b15239f4ac to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f4395c9376c04496b3c532b15239f4ac 2024-11-11T07:22:36,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/325f622680934044bf477167c368b1f6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/325f622680934044bf477167c368b1f6 2024-11-11T07:22:36,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/abf9329f1b8c4010bc87ed676fc50248 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/abf9329f1b8c4010bc87ed676fc50248 2024-11-11T07:22:36,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/72863b2b5c7b4439abe5dcee6a935b3c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/72863b2b5c7b4439abe5dcee6a935b3c 2024-11-11T07:22:36,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f0b3db212fa74ca590ac54e7c6acf8a9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f0b3db212fa74ca590ac54e7c6acf8a9 2024-11-11T07:22:36,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f85be4ef0e21433083daa79ccf7f37e6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f85be4ef0e21433083daa79ccf7f37e6 2024-11-11T07:22:36,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f98d8303b8864b2cb25cc8834fbc630f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/f98d8303b8864b2cb25cc8834fbc630f 2024-11-11T07:22:36,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b14724340dc04cd8a632a128007cc8b9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b14724340dc04cd8a632a128007cc8b9 2024-11-11T07:22:36,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4fbf2ac53d7a4f209d47d25ddc379772 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4fbf2ac53d7a4f209d47d25ddc379772 2024-11-11T07:22:36,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9943559c00cb462486f969082100d955 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/9943559c00cb462486f969082100d955 2024-11-11T07:22:36,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4eb3b07feef74466839135a03caf4398 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/4eb3b07feef74466839135a03caf4398 2024-11-11T07:22:36,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/de3b3f94d0ab413884c008353844a068 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/de3b3f94d0ab413884c008353844a068 2024-11-11T07:22:36,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/13995f07c4d6474e9f541dddab5ba6f4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/13995f07c4d6474e9f541dddab5ba6f4 2024-11-11T07:22:36,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/303f5a32862244ce8a0b1d7a2a9e5b88 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/303f5a32862244ce8a0b1d7a2a9e5b88 2024-11-11T07:22:36,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdb50ce3135e4410a92c6b4ae811372c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdb50ce3135e4410a92c6b4ae811372c 2024-11-11T07:22:36,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c5e07633d2d44957b2a75bd99fcf40cf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c5e07633d2d44957b2a75bd99fcf40cf 2024-11-11T07:22:36,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/3e4d223d07ea4c768916449656f1c2a7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/3e4d223d07ea4c768916449656f1c2a7 2024-11-11T07:22:36,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7e7d7724057d48919e1a6fc9a1f9114c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/7e7d7724057d48919e1a6fc9a1f9114c 2024-11-11T07:22:36,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b70aa1d09ed046ec8e47dd320b91d3cc to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/b70aa1d09ed046ec8e47dd320b91d3cc 2024-11-11T07:22:36,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/69205063d25b43baa48f099ac90a353d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/69205063d25b43baa48f099ac90a353d 2024-11-11T07:22:36,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/5a21a12c25b84134bb890606f47464a7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/5a21a12c25b84134bb890606f47464a7 2024-11-11T07:22:36,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c1ceb43b35d24b4db97827dd481ee35a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c1ceb43b35d24b4db97827dd481ee35a 2024-11-11T07:22:36,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/02aab7f5f2ab43809aa7590564c57070 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/02aab7f5f2ab43809aa7590564c57070 2024-11-11T07:22:36,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/6b9658f37b4d4cefbe65dc59bdd14d8b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/6b9658f37b4d4cefbe65dc59bdd14d8b 2024-11-11T07:22:36,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/405b6425a882475080b3748e8aa2235e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/405b6425a882475080b3748e8aa2235e 2024-11-11T07:22:36,449 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=1 2024-11-11T07:22:36,450 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. 2024-11-11T07:22:36,450 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 5a2be9895ea70c6a849961e6f35c48df: Waiting for close lock at 1731309754377Running coprocessor pre-close hooks at 1731309754377Disabling compacts and flushes for region at 1731309754377Disabling writes for close at 1731309755022 (+645 ms)Obtaining lock to block concurrent updates at 1731309755022Preparing flush snapshotting stores in 5a2be9895ea70c6a849961e6f35c48df at 1731309755022Finished memstore snapshotting TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731309755022Flushing stores of TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df. at 1731309755023 (+1 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/A: creating writer at 1731309755023Flushing 5a2be9895ea70c6a849961e6f35c48df/A: appending metadata at 1731309755025 (+2 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/A: closing flushed file at 1731309755026 (+1 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/B: creating writer at 1731309755438 (+412 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/B: appending metadata at 1731309755440 (+2 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/B: closing flushed file at 1731309755441 (+1 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/C: creating writer at 1731309755850 (+409 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/C: appending metadata at 1731309755853 (+3 ms)Flushing 5a2be9895ea70c6a849961e6f35c48df/C: closing flushed file at 1731309755853Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@538769d1: reopening flushed file at 1731309756263 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42a6098f: reopening flushed file at 1731309756268 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38be433: reopening flushed file at 1731309756273 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 5a2be9895ea70c6a849961e6f35c48df in 1257ms, sequenceid=544, compaction requested=true at 1731309756279 (+6 ms)Writing region close event to WAL at 1731309756446 (+167 ms)Running coprocessor post-close hooks at 1731309756450 (+4 ms)Closed at 1731309756450 2024-11-11T07:22:36,453 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:36,453 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=5a2be9895ea70c6a849961e6f35c48df, regionState=CLOSED 2024-11-11T07:22:36,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:22:36,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T07:22:36,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 5a2be9895ea70c6a849961e6f35c48df, server=f334e8b0fbdf,39329,1731309728191 in 2.2370 sec 2024-11-11T07:22:36,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-11T07:22:36,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a2be9895ea70c6a849961e6f35c48df, UNASSIGN in 2.2430 sec 2024-11-11T07:22:36,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T07:22:36,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2490 sec 2024-11-11T07:22:36,462 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309756462"}]},"ts":"1731309756462"} 2024-11-11T07:22:36,464 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:22:36,464 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:22:36,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2630 sec 2024-11-11T07:22:36,528 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:22:37,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T07:22:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T07:22:38,361 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:38,365 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:22:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:38,370 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T07:22:38,372 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:38,376 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:38,379 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/recovered.edits] 2024-11-11T07:22:38,382 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/09439a099ed241189e462925a2a56ace to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/09439a099ed241189e462925a2a56ace 2024-11-11T07:22:38,384 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/176e582021954d35b843b2fcda073321 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/176e582021954d35b843b2fcda073321 2024-11-11T07:22:38,385 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/69362cc70aef499ab4dcd8a461bfac37 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/69362cc70aef499ab4dcd8a461bfac37 2024-11-11T07:22:38,387 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/af63a87a94544e38952b47c881a22132 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/A/af63a87a94544e38952b47c881a22132 2024-11-11T07:22:38,389 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/19345572e52d4fd3bfa06ed47dc73506 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/19345572e52d4fd3bfa06ed47dc73506 2024-11-11T07:22:38,391 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49c37ab4571945058104b3193e88d357 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/49c37ab4571945058104b3193e88d357 2024-11-11T07:22:38,392 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/4a690b086f644b248b48cab27feb035e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/4a690b086f644b248b48cab27feb035e 2024-11-11T07:22:38,393 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/d7d74dd9fbad4e50a101bb89c5601cc9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/B/d7d74dd9fbad4e50a101bb89c5601cc9 2024-11-11T07:22:38,396 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/1505877823ed46ceab3be7926f907b56 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/1505877823ed46ceab3be7926f907b56 2024-11-11T07:22:38,397 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a8617a72fc55404fab1d525094cea823 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/a8617a72fc55404fab1d525094cea823 2024-11-11T07:22:38,398 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdf11220d3574b788741b4b871ff3bfa to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/bdf11220d3574b788741b4b871ff3bfa 2024-11-11T07:22:38,399 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c2cdb360772c4f1ba950d84668c167b8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/C/c2cdb360772c4f1ba950d84668c167b8 2024-11-11T07:22:38,402 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/recovered.edits/547.seqid to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df/recovered.edits/547.seqid 2024-11-11T07:22:38,403 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/5a2be9895ea70c6a849961e6f35c48df 2024-11-11T07:22:38,403 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:22:38,409 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:38,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T07:22:38,418 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:22:38,448 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:22:38,450 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:38,450 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:22:38,450 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731309758450"}]},"ts":"9223372036854775807"} 2024-11-11T07:22:38,455 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:22:38,455 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5a2be9895ea70c6a849961e6f35c48df, NAME => 'TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:22:38,455 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:22:38,455 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731309758455"}]},"ts":"9223372036854775807"} 2024-11-11T07:22:38,458 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:22:38,459 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:38,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-11-11T07:22:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T07:22:38,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:22:38,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:38,495 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f334e8b0fbdf:39329-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-21932362_22 at /127.0.0.1:45568 [Waiting for operation #270] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1782211046_22 at /127.0.0.1:33130 [Waiting for operation #249] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1782211046_22 at /127.0.0.1:33146 [Waiting for operation #250] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=326 (was 188) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8338 (was 8893) 2024-11-11T07:22:38,504 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=326, ProcessCount=11, AvailableMemoryMB=8338 2024-11-11T07:22:38,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:22:38,506 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:22:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:38,508 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:22:38,508 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:38,509 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-11T07:22:38,510 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:22:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T07:22:38,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741960_1136 (size=959) 2024-11-11T07:22:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T07:22:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T07:22:38,918 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:22:38,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741961_1137 (size=53) 2024-11-11T07:22:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T07:22:39,324 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:39,324 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8315bbc46c6a14c3758164d93d7c2260, disabling compactions & flushes 2024-11-11T07:22:39,324 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:39,324 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:39,324 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. after waiting 0 ms 2024-11-11T07:22:39,324 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:39,324 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:39,324 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8315bbc46c6a14c3758164d93d7c2260: Waiting for close lock at 1731309759324Disabling compacts and flushes for region at 1731309759324Disabling writes for close at 1731309759324Writing region close event to WAL at 1731309759324Closed at 1731309759324 2024-11-11T07:22:39,325 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:22:39,326 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731309759325"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731309759325"}]},"ts":"1731309759325"} 2024-11-11T07:22:39,327 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:22:39,328 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:22:39,328 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309759328"}]},"ts":"1731309759328"} 2024-11-11T07:22:39,330 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:22:39,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, ASSIGN}] 2024-11-11T07:22:39,331 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, ASSIGN 2024-11-11T07:22:39,331 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, ASSIGN; state=OFFLINE, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=false 2024-11-11T07:22:39,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:39,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, ASSIGN because future has completed 2024-11-11T07:22:39,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:22:39,640 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:39,641 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:22:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T07:22:39,641 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,641 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:39,641 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,642 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,643 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,645 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8315bbc46c6a14c3758164d93d7c2260 columnFamilyName A 2024-11-11T07:22:39,645 DEBUG [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:39,646 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(327): Store=8315bbc46c6a14c3758164d93d7c2260/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:39,646 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,647 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8315bbc46c6a14c3758164d93d7c2260 columnFamilyName B 2024-11-11T07:22:39,647 DEBUG [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:39,647 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(327): Store=8315bbc46c6a14c3758164d93d7c2260/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:39,647 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,648 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8315bbc46c6a14c3758164d93d7c2260 columnFamilyName C 2024-11-11T07:22:39,648 DEBUG [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:39,649 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(327): Store=8315bbc46c6a14c3758164d93d7c2260/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:39,649 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,649 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,650 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,650 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,650 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,651 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:22:39,652 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,653 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:22:39,654 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 8315bbc46c6a14c3758164d93d7c2260; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70752354, jitterRate=0.054292231798172}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:22:39,654 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:39,655 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 8315bbc46c6a14c3758164d93d7c2260: Running coprocessor pre-open hook at 1731309759642Writing region info on filesystem at 1731309759642Initializing all the Stores at 1731309759642Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309759643 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309759643Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309759643Cleaning up temporary data from old regions at 1731309759650 (+7 ms)Running coprocessor post-open hooks at 1731309759654 (+4 ms)Region opened successfully at 1731309759655 (+1 ms) 2024-11-11T07:22:39,656 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., pid=36, masterSystemTime=1731309759637 2024-11-11T07:22:39,658 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:39,658 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:39,658 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=OPEN, openSeqNum=2, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:39,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:22:39,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T07:22:39,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 in 192 msec 2024-11-11T07:22:39,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T07:22:39,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, ASSIGN in 349 msec 2024-11-11T07:22:39,681 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:22:39,681 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309759681"}]},"ts":"1731309759681"} 2024-11-11T07:22:39,683 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:22:39,683 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:22:39,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-11T07:22:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T07:22:40,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:40,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:40,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:40,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:40,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743a41ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:40,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f1990d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:40,657 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:40,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:40,661 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:22:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f334e8b0fbdf,39865,1731309727465 2024-11-11T07:22:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@31cb7c47 2024-11-11T07:22:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:22:40,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:22:40,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:22:40,668 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:22:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T07:22:40,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741962_1138 (size=995) 2024-11-11T07:22:41,087 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T07:22:41,087 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T07:22:41,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:22:41,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, REOPEN/MOVE}] 2024-11-11T07:22:41,098 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, REOPEN/MOVE 2024-11-11T07:22:41,098 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, REOPEN/MOVE because future has completed 2024-11-11T07:22:41,100 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:22:41,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:22:41,253 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,253 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:22:41,253 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 8315bbc46c6a14c3758164d93d7c2260, disabling compactions & flushes 2024-11-11T07:22:41,253 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,253 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,253 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. after waiting 0 ms 2024-11-11T07:22:41,253 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,257 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T07:22:41,258 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,258 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 8315bbc46c6a14c3758164d93d7c2260: Waiting for close lock at 1731309761253Running coprocessor pre-close hooks at 1731309761253Disabling compacts and flushes for region at 1731309761253Disabling writes for close at 1731309761253Writing region close event to WAL at 1731309761254 (+1 ms)Running coprocessor post-close hooks at 1731309761258 (+4 ms)Closed at 1731309761258 2024-11-11T07:22:41,258 WARN [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 8315bbc46c6a14c3758164d93d7c2260 to self. 2024-11-11T07:22:41,260 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,260 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=CLOSED 2024-11-11T07:22:41,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:22:41,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-11T07:22:41,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 in 162 msec 2024-11-11T07:22:41,265 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, REOPEN/MOVE; state=CLOSED, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=true 2024-11-11T07:22:41,416 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, REOPEN/MOVE because future has completed 2024-11-11T07:22:41,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:22:41,573 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,573 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:22:41,574 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,574 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:22:41,574 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,574 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,579 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,584 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8315bbc46c6a14c3758164d93d7c2260 columnFamilyName A 2024-11-11T07:22:41,586 DEBUG [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:41,586 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(327): Store=8315bbc46c6a14c3758164d93d7c2260/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:41,587 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,588 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8315bbc46c6a14c3758164d93d7c2260 columnFamilyName B 2024-11-11T07:22:41,588 DEBUG [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:41,588 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(327): Store=8315bbc46c6a14c3758164d93d7c2260/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:41,588 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,589 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8315bbc46c6a14c3758164d93d7c2260 columnFamilyName C 2024-11-11T07:22:41,589 DEBUG [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:41,589 INFO [StoreOpener-8315bbc46c6a14c3758164d93d7c2260-1 {}] regionserver.HStore(327): Store=8315bbc46c6a14c3758164d93d7c2260/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:22:41,590 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,590 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,591 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,592 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,592 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,593 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:22:41,594 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,595 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 8315bbc46c6a14c3758164d93d7c2260; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62244982, jitterRate=-0.07247748970985413}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:22:41,595 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,596 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 8315bbc46c6a14c3758164d93d7c2260: Running coprocessor pre-open hook at 1731309761574Writing region info on filesystem at 1731309761574Initializing all the Stores at 1731309761575 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309761575Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309761575Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309761575Cleaning up temporary data from old regions at 1731309761593 (+18 ms)Running coprocessor post-open hooks at 1731309761595 (+2 ms)Region opened successfully at 1731309761596 (+1 ms) 2024-11-11T07:22:41,597 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., pid=41, masterSystemTime=1731309761569 2024-11-11T07:22:41,600 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,600 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,600 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=OPEN, openSeqNum=5, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:22:41,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-11T07:22:41,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 in 185 msec 2024-11-11T07:22:41,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T07:22:41,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, REOPEN/MOVE in 507 msec 2024-11-11T07:22:41,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-11T07:22:41,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-11T07:22:41,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-11T07:22:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-11T07:22:41,614 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:41,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4febd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,618 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d767205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,620 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,622 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cdba98c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,624 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@193fee02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,626 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179fd5c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,628 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd1ce3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a8fd72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,631 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7550e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,633 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@786fc3d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254ed396, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ff15db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,639 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a776fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,641 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516cb54e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,643 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d661940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,644 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739a3af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:22:41,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:22:41,646 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:22:41,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:22:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:22:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265983ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:22:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:22:41,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:22:41,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:22:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1358217d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:22:41,649 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,649 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,649 DEBUG [Thread-690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,650 DEBUG [Thread-692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,651 DEBUG [Thread-701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,661 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,661 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:41,662 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T07:22:41,663 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,663 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:22:41,663 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:22:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:22:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:22:41,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,666 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:41,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:41,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:22:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:22:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,675 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:41,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:22:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309821710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309821711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309821715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309821713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309821719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111597f13d3d8664f479ac4b171bcb83445_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309761672/Put/seqid=0 2024-11-11T07:22:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741963_1139 (size=14594) 2024-11-11T07:22:41,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:41,763 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111597f13d3d8664f479ac4b171bcb83445_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111597f13d3d8664f479ac4b171bcb83445_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:41,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/03ecf29b26494def89d26ba775143e39, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:22:41,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/03ecf29b26494def89d26ba775143e39 is 175, key is test_row_0/A:col10/1731309761672/Put/seqid=0 2024-11-11T07:22:41,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741964_1140 (size=39549) 2024-11-11T07:22:41,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/03ecf29b26494def89d26ba775143e39 2024-11-11T07:22:41,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:41,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e6e7794a5e454f609dce28ce835f3b97 is 50, key is test_row_0/B:col10/1731309761672/Put/seqid=0 2024-11-11T07:22:41,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309821833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309821833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309821832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309821833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309821833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:41,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741965_1141 (size=12001) 2024-11-11T07:22:41,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e6e7794a5e454f609dce28ce835f3b97 2024-11-11T07:22:41,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/ff0d548b3cf648a698f46665295fc7bd is 50, key is test_row_0/C:col10/1731309761672/Put/seqid=0 2024-11-11T07:22:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741966_1142 (size=12001) 2024-11-11T07:22:41,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:41,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:41,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:41,978 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:22:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:22:42,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309822052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309822052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309822053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309822053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309822053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:42,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:42,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:22:42,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/ff0d548b3cf648a698f46665295fc7bd 2024-11-11T07:22:42,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/03ecf29b26494def89d26ba775143e39 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/03ecf29b26494def89d26ba775143e39 2024-11-11T07:22:42,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/03ecf29b26494def89d26ba775143e39, entries=200, sequenceid=17, filesize=38.6 K 2024-11-11T07:22:42,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e6e7794a5e454f609dce28ce835f3b97 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e6e7794a5e454f609dce28ce835f3b97 2024-11-11T07:22:42,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e6e7794a5e454f609dce28ce835f3b97, entries=150, sequenceid=17, filesize=11.7 K 2024-11-11T07:22:42,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/ff0d548b3cf648a698f46665295fc7bd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/ff0d548b3cf648a698f46665295fc7bd 2024-11-11T07:22:42,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/ff0d548b3cf648a698f46665295fc7bd, entries=150, sequenceid=17, filesize=11.7 K 2024-11-11T07:22:42,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8315bbc46c6a14c3758164d93d7c2260 in 668ms, sequenceid=17, compaction requested=false 2024-11-11T07:22:42,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:42,369 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:42,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:22:42,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111169d566fda11b49cf85ec1abe6f42cf2d_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309762367/Put/seqid=0 2024-11-11T07:22:42,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741967_1143 (size=17034) 2024-11-11T07:22:42,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309822414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309822412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309822420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309822428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309822430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:42,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:42,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309822532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309822541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309822541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309822541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309822542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:42,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:42,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309822742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:42,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:42,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309822752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309822752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309822755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309822755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:42,793 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:42,800 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111169d566fda11b49cf85ec1abe6f42cf2d_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169d566fda11b49cf85ec1abe6f42cf2d_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:22:42,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/3ae3b918af1447df8f0a5490e352fda2, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:42,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/3ae3b918af1447df8f0a5490e352fda2 is 175, key is test_row_0/A:col10/1731309762367/Put/seqid=0 2024-11-11T07:22:42,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741968_1144 (size=48139) 2024-11-11T07:22:42,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/3ae3b918af1447df8f0a5490e352fda2 2024-11-11T07:22:42,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/06e5dcc5819443e0a1e3899cebc434f8 is 50, key is test_row_0/B:col10/1731309762367/Put/seqid=0 2024-11-11T07:22:42,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741969_1145 (size=12001) 2024-11-11T07:22:42,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:42,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:42,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:42,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309823051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:43,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:43,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:43,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:43,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:43,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309823062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309823062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309823062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309823062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:43,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:43,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/06e5dcc5819443e0a1e3899cebc434f8 2024-11-11T07:22:43,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/c63520948d69458da12bbabbe2a93633 is 50, key is test_row_0/C:col10/1731309762367/Put/seqid=0 2024-11-11T07:22:43,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741970_1146 (size=12001) 2024-11-11T07:22:43,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/c63520948d69458da12bbabbe2a93633 2024-11-11T07:22:43,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/3ae3b918af1447df8f0a5490e352fda2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3ae3b918af1447df8f0a5490e352fda2 2024-11-11T07:22:43,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3ae3b918af1447df8f0a5490e352fda2, entries=250, sequenceid=42, filesize=47.0 K 2024-11-11T07:22:43,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/06e5dcc5819443e0a1e3899cebc434f8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/06e5dcc5819443e0a1e3899cebc434f8 2024-11-11T07:22:43,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/06e5dcc5819443e0a1e3899cebc434f8, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T07:22:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/c63520948d69458da12bbabbe2a93633 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/c63520948d69458da12bbabbe2a93633 2024-11-11T07:22:43,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/c63520948d69458da12bbabbe2a93633, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T07:22:43,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8315bbc46c6a14c3758164d93d7c2260 in 968ms, sequenceid=42, compaction requested=false 2024-11-11T07:22:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:43,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:22:43,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:43,374 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ec018406a65144e8ae1ba00c7b6ef58b_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309762425/Put/seqid=0 2024-11-11T07:22:43,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741971_1147 (size=12154) 2024-11-11T07:22:43,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:43,450 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ec018406a65144e8ae1ba00c7b6ef58b_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ec018406a65144e8ae1ba00c7b6ef58b_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:43,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/573b1a2190c148698c4b93820d6d4e8c, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/573b1a2190c148698c4b93820d6d4e8c is 175, key is test_row_0/A:col10/1731309762425/Put/seqid=0 2024-11-11T07:22:43,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741972_1148 (size=30955) 2024-11-11T07:22:43,469 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/573b1a2190c148698c4b93820d6d4e8c 2024-11-11T07:22:43,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/ecf3aaaa9f79479aad3ac63d30b2d83c is 50, key is test_row_0/B:col10/1731309762425/Put/seqid=0 2024-11-11T07:22:43,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741973_1149 (size=12001) 2024-11-11T07:22:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:43,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:43,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309823630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309823634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309823634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309823638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309823646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309823751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309823762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309823762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309823763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309823763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:22:43,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/ecf3aaaa9f79479aad3ac63d30b2d83c 2024-11-11T07:22:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/705b9e3e30a343848be83505d34676bc is 50, key is test_row_0/C:col10/1731309762425/Put/seqid=0 2024-11-11T07:22:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309823961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741974_1150 (size=12001) 2024-11-11T07:22:43,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309823972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309823972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309823973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309823973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309824272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309824281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309824281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309824282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309824293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,368 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/705b9e3e30a343848be83505d34676bc 2024-11-11T07:22:44,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/573b1a2190c148698c4b93820d6d4e8c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/573b1a2190c148698c4b93820d6d4e8c 2024-11-11T07:22:44,390 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/573b1a2190c148698c4b93820d6d4e8c, entries=150, sequenceid=53, filesize=30.2 K 2024-11-11T07:22:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/ecf3aaaa9f79479aad3ac63d30b2d83c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ecf3aaaa9f79479aad3ac63d30b2d83c 2024-11-11T07:22:44,404 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ecf3aaaa9f79479aad3ac63d30b2d83c, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T07:22:44,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/705b9e3e30a343848be83505d34676bc as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/705b9e3e30a343848be83505d34676bc 2024-11-11T07:22:44,420 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/705b9e3e30a343848be83505d34676bc, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T07:22:44,424 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8315bbc46c6a14c3758164d93d7c2260 in 1049ms, sequenceid=53, compaction requested=true 2024-11-11T07:22:44,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:44,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:44,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T07:22:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T07:22:44,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T07:22:44,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7640 sec 2024-11-11T07:22:44,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.7760 sec 2024-11-11T07:22:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:44,785 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:44,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T07:22:44,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111281db33b6a284d0b97e70d98882e4de0_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309763631/Put/seqid=0 2024-11-11T07:22:44,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741975_1151 (size=14594) 2024-11-11T07:22:44,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309824826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309824826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309824826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309824834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309824834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309824941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309824942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309824942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309824942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:44,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309824951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309825152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309825152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309825152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309825152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309825173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,219 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:45,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111281db33b6a284d0b97e70d98882e4de0_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111281db33b6a284d0b97e70d98882e4de0_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:45,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/0f33a9b8b0ae4469875e3b6a724b6e56, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:45,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/0f33a9b8b0ae4469875e3b6a724b6e56 is 175, key is test_row_0/A:col10/1731309763631/Put/seqid=0 2024-11-11T07:22:45,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741976_1152 (size=39549) 2024-11-11T07:22:45,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309825461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309825473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309825473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309825474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309825497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,661 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/0f33a9b8b0ae4469875e3b6a724b6e56 2024-11-11T07:22:45,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/32000795dba64d7dad91e5ca0405138d is 50, key is test_row_0/B:col10/1731309763631/Put/seqid=0 2024-11-11T07:22:45,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741977_1153 (size=12001) 2024-11-11T07:22:45,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/32000795dba64d7dad91e5ca0405138d 2024-11-11T07:22:45,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/d65ee4b1e9974d9bbce71d54380d95d6 is 50, key is test_row_0/C:col10/1731309763631/Put/seqid=0 2024-11-11T07:22:45,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741978_1154 (size=12001) 2024-11-11T07:22:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:22:45,829 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:45,831 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T07:22:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:22:45,835 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:45,836 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:45,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:22:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309825972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:45,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T07:22:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:45,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:46,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309826001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309826001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:46,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309826012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309826012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:46,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/d65ee4b1e9974d9bbce71d54380d95d6 2024-11-11T07:22:46,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/0f33a9b8b0ae4469875e3b6a724b6e56 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/0f33a9b8b0ae4469875e3b6a724b6e56 2024-11-11T07:22:46,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/0f33a9b8b0ae4469875e3b6a724b6e56, entries=200, sequenceid=79, filesize=38.6 K 2024-11-11T07:22:46,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/32000795dba64d7dad91e5ca0405138d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/32000795dba64d7dad91e5ca0405138d 2024-11-11T07:22:46,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T07:22:46,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:46,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:46,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:22:46,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/32000795dba64d7dad91e5ca0405138d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T07:22:46,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/d65ee4b1e9974d9bbce71d54380d95d6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/d65ee4b1e9974d9bbce71d54380d95d6 2024-11-11T07:22:46,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/d65ee4b1e9974d9bbce71d54380d95d6, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T07:22:46,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 8315bbc46c6a14c3758164d93d7c2260 in 1376ms, sequenceid=79, compaction requested=true 2024-11-11T07:22:46,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:46,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:46,162 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:46,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:46,162 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:46,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:46,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:46,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:46,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:46,163 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:46,163 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/A is initiating minor compaction (all files) 2024-11-11T07:22:46,163 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/A in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,163 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/03ecf29b26494def89d26ba775143e39, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3ae3b918af1447df8f0a5490e352fda2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/573b1a2190c148698c4b93820d6d4e8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/0f33a9b8b0ae4469875e3b6a724b6e56] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=154.5 K 2024-11-11T07:22:46,163 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,163 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/03ecf29b26494def89d26ba775143e39, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3ae3b918af1447df8f0a5490e352fda2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/573b1a2190c148698c4b93820d6d4e8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/0f33a9b8b0ae4469875e3b6a724b6e56] 2024-11-11T07:22:46,164 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:46,164 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ecf29b26494def89d26ba775143e39, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731309761668 2024-11-11T07:22:46,164 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/B is initiating minor compaction (all files) 2024-11-11T07:22:46,164 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/B in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,164 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e6e7794a5e454f609dce28ce835f3b97, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/06e5dcc5819443e0a1e3899cebc434f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ecf3aaaa9f79479aad3ac63d30b2d83c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/32000795dba64d7dad91e5ca0405138d] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=46.9 K 2024-11-11T07:22:46,165 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae3b918af1447df8f0a5490e352fda2, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731309761708 2024-11-11T07:22:46,166 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e7794a5e454f609dce28ce835f3b97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731309761672 2024-11-11T07:22:46,166 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573b1a2190c148698c4b93820d6d4e8c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309762411 2024-11-11T07:22:46,168 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e5dcc5819443e0a1e3899cebc434f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731309761710 2024-11-11T07:22:46,168 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f33a9b8b0ae4469875e3b6a724b6e56, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731309763631 2024-11-11T07:22:46,168 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting ecf3aaaa9f79479aad3ac63d30b2d83c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309762411 2024-11-11T07:22:46,170 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 32000795dba64d7dad91e5ca0405138d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731309763631 2024-11-11T07:22:46,189 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:46,196 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118bd7e6fb653f4f8898ea1a4ea9d1fbb0_8315bbc46c6a14c3758164d93d7c2260 store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:46,196 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#B#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:46,197 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/7d2334a7be1542129da6aac01bb3a2ad is 50, key is test_row_0/B:col10/1731309763631/Put/seqid=0 2024-11-11T07:22:46,206 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118bd7e6fb653f4f8898ea1a4ea9d1fbb0_8315bbc46c6a14c3758164d93d7c2260, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:46,206 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118bd7e6fb653f4f8898ea1a4ea9d1fbb0_8315bbc46c6a14c3758164d93d7c2260 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:46,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741979_1155 (size=12139) 2024-11-11T07:22:46,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741980_1156 (size=4469) 2024-11-11T07:22:46,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T07:22:46,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,305 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:46,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111140034bc6975b40519446d525dfe794ea_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309764830/Put/seqid=0 2024-11-11T07:22:46,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741981_1157 (size=12154) 2024-11-11T07:22:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,357 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111140034bc6975b40519446d525dfe794ea_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111140034bc6975b40519446d525dfe794ea_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:46,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/ac97c97d2aca4da0a7617ae38bc7725a, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:46,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/ac97c97d2aca4da0a7617ae38bc7725a is 175, key is test_row_0/A:col10/1731309764830/Put/seqid=0 2024-11-11T07:22:46,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741982_1158 (size=30955) 2024-11-11T07:22:46,375 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/ac97c97d2aca4da0a7617ae38bc7725a 2024-11-11T07:22:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/a923213afa154c5e8b380e070b7cfbf8 is 50, key is test_row_0/B:col10/1731309764830/Put/seqid=0 2024-11-11T07:22:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741983_1159 (size=12001) 2024-11-11T07:22:46,407 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/a923213afa154c5e8b380e070b7cfbf8 2024-11-11T07:22:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4 is 50, key is test_row_0/C:col10/1731309764830/Put/seqid=0 2024-11-11T07:22:46,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741984_1160 (size=12001) 2024-11-11T07:22:46,451 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4 2024-11-11T07:22:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/ac97c97d2aca4da0a7617ae38bc7725a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ac97c97d2aca4da0a7617ae38bc7725a 2024-11-11T07:22:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:22:46,463 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ac97c97d2aca4da0a7617ae38bc7725a, entries=150, sequenceid=90, filesize=30.2 K 2024-11-11T07:22:46,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/a923213afa154c5e8b380e070b7cfbf8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a923213afa154c5e8b380e070b7cfbf8 2024-11-11T07:22:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,475 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a923213afa154c5e8b380e070b7cfbf8, entries=150, sequenceid=90, filesize=11.7 K 2024-11-11T07:22:46,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4 2024-11-11T07:22:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,485 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4, entries=150, sequenceid=90, filesize=11.7 K 2024-11-11T07:22:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,486 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 8315bbc46c6a14c3758164d93d7c2260 in 181ms, sequenceid=90, compaction requested=true 2024-11-11T07:22:46,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:46,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T07:22:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T07:22:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T07:22:46,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 652 msec 2024-11-11T07:22:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 670 msec 2024-11-11T07:22:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,644 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#A#compaction#135 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,645 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/d1caaea6126f4623b94b2ed57521ffd6 is 175, key is test_row_0/A:col10/1731309763631/Put/seqid=0 2024-11-11T07:22:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,646 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/7d2334a7be1542129da6aac01bb3a2ad as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7d2334a7be1542129da6aac01bb3a2ad 2024-11-11T07:22:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,653 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/B of 8315bbc46c6a14c3758164d93d7c2260 into 7d2334a7be1542129da6aac01bb3a2ad(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:46,653 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:46,653 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/B, priority=12, startTime=1731309766162; duration=0sec 2024-11-11T07:22:46,653 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:46,653 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:B 2024-11-11T07:22:46,653 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T07:22:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,655 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T07:22:46,655 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/C is initiating minor compaction (all files) 2024-11-11T07:22:46,655 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/C in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:46,655 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/ff0d548b3cf648a698f46665295fc7bd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/c63520948d69458da12bbabbe2a93633, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/705b9e3e30a343848be83505d34676bc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/d65ee4b1e9974d9bbce71d54380d95d6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=58.6 K 2024-11-11T07:22:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,656 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting ff0d548b3cf648a698f46665295fc7bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731309761672 2024-11-11T07:22:46,656 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c63520948d69458da12bbabbe2a93633, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731309761710 2024-11-11T07:22:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,657 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 705b9e3e30a343848be83505d34676bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309762411 2024-11-11T07:22:46,657 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting d65ee4b1e9974d9bbce71d54380d95d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731309763631 2024-11-11T07:22:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,658 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e5c5bdd6eb4c6380d8952dfe92e6e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309764820 2024-11-11T07:22:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741985_1161 (size=31093) 2024-11-11T07:22:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,701 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/d1caaea6126f4623b94b2ed57521ffd6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/d1caaea6126f4623b94b2ed57521ffd6 2024-11-11T07:22:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,703 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:46,704 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/e6700cbcf8d74d3c855a9541bb71f28b is 50, key is test_row_0/C:col10/1731309764830/Put/seqid=0 2024-11-11T07:22:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,709 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/A of 8315bbc46c6a14c3758164d93d7c2260 into d1caaea6126f4623b94b2ed57521ffd6(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:46,709 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:46,709 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/A, priority=12, startTime=1731309766161; duration=0sec 2024-11-11T07:22:46,709 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:46,709 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:A 2024-11-11T07:22:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741986_1162 (size=12173) 2024-11-11T07:22:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,751 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/e6700cbcf8d74d3c855a9541bb71f28b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e6700cbcf8d74d3c855a9541bb71f28b 2024-11-11T07:22:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,762 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/C of 8315bbc46c6a14c3758164d93d7c2260 into e6700cbcf8d74d3c855a9541bb71f28b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:46,762 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:46,762 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/C, priority=11, startTime=1731309766162; duration=0sec 2024-11-11T07:22:46,762 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:46,762 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:C 2024-11-11T07:22:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:22:46,974 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T07:22:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,977 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:22:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,978 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:46,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:47,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:47,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,076 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:22:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:22:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ac31508a3c274850a72ea9d85ee1a376_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309767050/Put/seqid=0 2024-11-11T07:22:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309827112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309827112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309827115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309827117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309827125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:22:47,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741987_1163 (size=26798) 2024-11-11T07:22:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309827231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309827232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309827232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309827242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309827242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:22:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:47,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:22:47,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:22:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309827442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309827442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309827443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309827452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309827452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,560 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:47,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ac31508a3c274850a72ea9d85ee1a376_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac31508a3c274850a72ea9d85ee1a376_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:47,569 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/c1b18000a053489c9ca7d64f30cc3308, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:47,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/c1b18000a053489c9ca7d64f30cc3308 is 175, key is test_row_0/A:col10/1731309767050/Put/seqid=0 2024-11-11T07:22:47,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741988_1164 (size=82585) 2024-11-11T07:22:47,573 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/c1b18000a053489c9ca7d64f30cc3308 2024-11-11T07:22:47,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:22:47,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:47,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:47,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/adf58054400c42cebbdaf38d7dad7141 is 50, key is test_row_0/B:col10/1731309767050/Put/seqid=0 2024-11-11T07:22:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:22:47,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741989_1165 (size=12001) 2024-11-11T07:22:47,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/adf58054400c42cebbdaf38d7dad7141 2024-11-11T07:22:47,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/67ada2d31ff54098bb3e2f0ee6a5d66b is 50, key is test_row_0/C:col10/1731309767050/Put/seqid=0 2024-11-11T07:22:47,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741990_1166 (size=12001) 2024-11-11T07:22:47,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/67ada2d31ff54098bb3e2f0ee6a5d66b 2024-11-11T07:22:47,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/c1b18000a053489c9ca7d64f30cc3308 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/c1b18000a053489c9ca7d64f30cc3308 2024-11-11T07:22:47,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/c1b18000a053489c9ca7d64f30cc3308, entries=450, sequenceid=105, filesize=80.6 K 2024-11-11T07:22:47,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/adf58054400c42cebbdaf38d7dad7141 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/adf58054400c42cebbdaf38d7dad7141 2024-11-11T07:22:47,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/adf58054400c42cebbdaf38d7dad7141, entries=150, sequenceid=105, filesize=11.7 K 2024-11-11T07:22:47,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/67ada2d31ff54098bb3e2f0ee6a5d66b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67ada2d31ff54098bb3e2f0ee6a5d66b 2024-11-11T07:22:47,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67ada2d31ff54098bb3e2f0ee6a5d66b, entries=150, sequenceid=105, filesize=11.7 K 2024-11-11T07:22:47,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8315bbc46c6a14c3758164d93d7c2260 in 641ms, sequenceid=105, compaction requested=true 2024-11-11T07:22:47,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:47,693 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:47,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:47,694 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:47,694 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144633 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:47,695 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/A is initiating minor compaction (all files) 2024-11-11T07:22:47,695 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/A in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,695 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/d1caaea6126f4623b94b2ed57521ffd6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ac97c97d2aca4da0a7617ae38bc7725a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/c1b18000a053489c9ca7d64f30cc3308] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=141.2 K 2024-11-11T07:22:47,695 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,695 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/d1caaea6126f4623b94b2ed57521ffd6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ac97c97d2aca4da0a7617ae38bc7725a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/c1b18000a053489c9ca7d64f30cc3308] 2024-11-11T07:22:47,695 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:47,695 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1caaea6126f4623b94b2ed57521ffd6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731309763631 2024-11-11T07:22:47,696 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/B is initiating minor compaction (all files) 2024-11-11T07:22:47,696 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/B in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,696 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7d2334a7be1542129da6aac01bb3a2ad, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a923213afa154c5e8b380e070b7cfbf8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/adf58054400c42cebbdaf38d7dad7141] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=35.3 K 2024-11-11T07:22:47,696 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2334a7be1542129da6aac01bb3a2ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731309763631 2024-11-11T07:22:47,696 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac97c97d2aca4da0a7617ae38bc7725a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309764820 2024-11-11T07:22:47,700 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a923213afa154c5e8b380e070b7cfbf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309764820 2024-11-11T07:22:47,700 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b18000a053489c9ca7d64f30cc3308, keycount=450, bloomtype=ROW, size=80.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731309766990 2024-11-11T07:22:47,703 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting adf58054400c42cebbdaf38d7dad7141, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731309767039 2024-11-11T07:22:47,726 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:47,726 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#B#compaction#144 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:47,727 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/17618920ef914f6ba4284cdfa6ce93b5 is 50, key is test_row_0/B:col10/1731309767050/Put/seqid=0 2024-11-11T07:22:47,731 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111f66c93fe50b24780a1bc7454ce33bc82_8315bbc46c6a14c3758164d93d7c2260 store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:47,735 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111f66c93fe50b24780a1bc7454ce33bc82_8315bbc46c6a14c3758164d93d7c2260, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:47,735 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f66c93fe50b24780a1bc7454ce33bc82_8315bbc46c6a14c3758164d93d7c2260 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:47,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:22:47,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:47,753 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:22:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:47,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741992_1168 (size=12241) 2024-11-11T07:22:47,765 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/17618920ef914f6ba4284cdfa6ce93b5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/17618920ef914f6ba4284cdfa6ce93b5 2024-11-11T07:22:47,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309827767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309827768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309827768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309827770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309827771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,776 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/B of 8315bbc46c6a14c3758164d93d7c2260 into 17618920ef914f6ba4284cdfa6ce93b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:47,776 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:47,776 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/B, priority=13, startTime=1731309767694; duration=0sec 2024-11-11T07:22:47,776 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:47,776 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:B 2024-11-11T07:22:47,776 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T07:22:47,778 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:22:47,778 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:22:47,779 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. because compaction request was cancelled 2024-11-11T07:22:47,779 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:C 2024-11-11T07:22:47,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741991_1167 (size=4469) 2024-11-11T07:22:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cde92f3f4f97491d8bff76df6a482381_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309767753/Put/seqid=0 2024-11-11T07:22:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741993_1169 (size=17134) 2024-11-11T07:22:47,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309827884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309827884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309827885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309827885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309827885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:47,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T07:22:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T07:22:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309828092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309828092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309828092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309828101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309828101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:22:48,213 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#A#compaction#145 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:48,214 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/ab2e328b35fa45f992be1e8937f544aa is 175, key is test_row_0/A:col10/1731309767050/Put/seqid=0 2024-11-11T07:22:48,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:48,250 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cde92f3f4f97491d8bff76df6a482381_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cde92f3f4f97491d8bff76df6a482381_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/2a9d7b35073946a4b7450ca790eb11ba, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:48,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/2a9d7b35073946a4b7450ca790eb11ba is 175, key is test_row_0/A:col10/1731309767753/Put/seqid=0 2024-11-11T07:22:48,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741994_1170 (size=31195) 2024-11-11T07:22:48,267 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/ab2e328b35fa45f992be1e8937f544aa as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ab2e328b35fa45f992be1e8937f544aa 2024-11-11T07:22:48,274 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/A of 8315bbc46c6a14c3758164d93d7c2260 into ab2e328b35fa45f992be1e8937f544aa(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:48,274 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:48,274 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/A, priority=13, startTime=1731309767693; duration=0sec 2024-11-11T07:22:48,274 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:48,274 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:A 2024-11-11T07:22:48,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741995_1171 (size=48239) 2024-11-11T07:22:48,279 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/2a9d7b35073946a4b7450ca790eb11ba 2024-11-11T07:22:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/057ec8ba381c49d481c3bb4755a9453f is 50, key is test_row_0/B:col10/1731309767753/Put/seqid=0 2024-11-11T07:22:48,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741996_1172 (size=12051) 2024-11-11T07:22:48,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309828412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309828412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309828412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309828412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309828412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,705 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/057ec8ba381c49d481c3bb4755a9453f 2024-11-11T07:22:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/5fd9500c493b44bf82c795faf68fa3b7 is 50, key is test_row_0/C:col10/1731309767753/Put/seqid=0 2024-11-11T07:22:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741997_1173 (size=12051) 2024-11-11T07:22:48,742 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/5fd9500c493b44bf82c795faf68fa3b7 2024-11-11T07:22:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/2a9d7b35073946a4b7450ca790eb11ba as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/2a9d7b35073946a4b7450ca790eb11ba 2024-11-11T07:22:48,757 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/2a9d7b35073946a4b7450ca790eb11ba, entries=250, sequenceid=131, filesize=47.1 K 2024-11-11T07:22:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/057ec8ba381c49d481c3bb4755a9453f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/057ec8ba381c49d481c3bb4755a9453f 2024-11-11T07:22:48,765 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/057ec8ba381c49d481c3bb4755a9453f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T07:22:48,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/5fd9500c493b44bf82c795faf68fa3b7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/5fd9500c493b44bf82c795faf68fa3b7 2024-11-11T07:22:48,772 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/5fd9500c493b44bf82c795faf68fa3b7, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T07:22:48,774 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8315bbc46c6a14c3758164d93d7c2260 in 1021ms, sequenceid=131, compaction requested=true 2024-11-11T07:22:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T07:22:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T07:22:48,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T07:22:48,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7980 sec 2024-11-11T07:22:48,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8030 sec 2024-11-11T07:22:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:48,928 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:48,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:22:48,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113dd4bf8179944d279685a01baa8a3913_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309768928/Put/seqid=0 2024-11-11T07:22:48,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741998_1174 (size=17284) 2024-11-11T07:22:48,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:48,972 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113dd4bf8179944d279685a01baa8a3913_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113dd4bf8179944d279685a01baa8a3913_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:48,974 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/eebc4565f6b840f491c6c78c5882b4f6, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:48,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/eebc4565f6b840f491c6c78c5882b4f6 is 175, key is test_row_0/A:col10/1731309768928/Put/seqid=0 2024-11-11T07:22:48,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309828965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309828973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309828974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309828977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309828978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:48,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741999_1175 (size=48389) 2024-11-11T07:22:49,001 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/eebc4565f6b840f491c6c78c5882b4f6 2024-11-11T07:22:49,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/ef07a3b5712e43ebb0513ca6e640ffe9 is 50, key is test_row_0/B:col10/1731309768928/Put/seqid=0 2024-11-11T07:22:49,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742000_1176 (size=12151) 2024-11-11T07:22:49,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/ef07a3b5712e43ebb0513ca6e640ffe9 2024-11-11T07:22:49,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/556da1bf4931452d81dce78dd4f96b5c is 50, key is test_row_0/C:col10/1731309768928/Put/seqid=0 2024-11-11T07:22:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309829081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309829081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309829093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309829093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309829094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742001_1177 (size=12151) 2024-11-11T07:22:49,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/556da1bf4931452d81dce78dd4f96b5c 2024-11-11T07:22:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:22:49,133 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:49,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:49,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/eebc4565f6b840f491c6c78c5882b4f6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/eebc4565f6b840f491c6c78c5882b4f6 2024-11-11T07:22:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T07:22:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:22:49,138 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:49,163 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:49,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:49,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/eebc4565f6b840f491c6c78c5882b4f6, entries=250, sequenceid=147, filesize=47.3 K 2024-11-11T07:22:49,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/ef07a3b5712e43ebb0513ca6e640ffe9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ef07a3b5712e43ebb0513ca6e640ffe9 2024-11-11T07:22:49,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ef07a3b5712e43ebb0513ca6e640ffe9, entries=150, sequenceid=147, filesize=11.9 K 2024-11-11T07:22:49,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/556da1bf4931452d81dce78dd4f96b5c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/556da1bf4931452d81dce78dd4f96b5c 2024-11-11T07:22:49,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/556da1bf4931452d81dce78dd4f96b5c, entries=150, sequenceid=147, filesize=11.9 K 2024-11-11T07:22:49,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8315bbc46c6a14c3758164d93d7c2260 in 253ms, sequenceid=147, compaction requested=true 2024-11-11T07:22:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:49,181 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:49,181 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:22:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:49,182 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:49,182 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:22:49,183 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/B is initiating minor compaction (all files) 2024-11-11T07:22:49,183 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/A is initiating minor compaction (all files) 2024-11-11T07:22:49,183 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/B in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,183 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/A in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,183 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/17618920ef914f6ba4284cdfa6ce93b5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/057ec8ba381c49d481c3bb4755a9453f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ef07a3b5712e43ebb0513ca6e640ffe9] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=35.6 K 2024-11-11T07:22:49,183 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ab2e328b35fa45f992be1e8937f544aa, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/2a9d7b35073946a4b7450ca790eb11ba, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/eebc4565f6b840f491c6c78c5882b4f6] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=124.8 K 2024-11-11T07:22:49,183 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,183 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ab2e328b35fa45f992be1e8937f544aa, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/2a9d7b35073946a4b7450ca790eb11ba, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/eebc4565f6b840f491c6c78c5882b4f6] 2024-11-11T07:22:49,183 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 17618920ef914f6ba4284cdfa6ce93b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731309767039 2024-11-11T07:22:49,184 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab2e328b35fa45f992be1e8937f544aa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731309767039 2024-11-11T07:22:49,184 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 057ec8ba381c49d481c3bb4755a9453f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309767114 2024-11-11T07:22:49,184 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a9d7b35073946a4b7450ca790eb11ba, keycount=250, bloomtype=ROW, size=47.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309767098 2024-11-11T07:22:49,184 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting ef07a3b5712e43ebb0513ca6e640ffe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731309767766 2024-11-11T07:22:49,185 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting eebc4565f6b840f491c6c78c5882b4f6, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731309767766 2024-11-11T07:22:49,203 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:49,204 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e1bccf5fa49c47d3a80d23fb8942bc67 is 50, key is test_row_0/B:col10/1731309768928/Put/seqid=0 2024-11-11T07:22:49,211 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:49,224 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112ff8bf4707324e3f900461262654fd80_8315bbc46c6a14c3758164d93d7c2260 store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:49,226 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112ff8bf4707324e3f900461262654fd80_8315bbc46c6a14c3758164d93d7c2260, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:49,227 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112ff8bf4707324e3f900461262654fd80_8315bbc46c6a14c3758164d93d7c2260 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:22:49,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742002_1178 (size=12493) 2024-11-11T07:22:49,259 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e1bccf5fa49c47d3a80d23fb8942bc67 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e1bccf5fa49c47d3a80d23fb8942bc67 2024-11-11T07:22:49,267 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/B of 8315bbc46c6a14c3758164d93d7c2260 into e1bccf5fa49c47d3a80d23fb8942bc67(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:49,267 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:49,267 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/B, priority=13, startTime=1731309769181; duration=0sec 2024-11-11T07:22:49,267 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:49,267 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:B 2024-11-11T07:22:49,267 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:49,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742003_1179 (size=4469) 2024-11-11T07:22:49,270 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:49,270 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/C is initiating minor compaction (all files) 2024-11-11T07:22:49,270 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/C in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,270 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e6700cbcf8d74d3c855a9541bb71f28b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67ada2d31ff54098bb3e2f0ee6a5d66b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/5fd9500c493b44bf82c795faf68fa3b7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/556da1bf4931452d81dce78dd4f96b5c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=47.2 K 2024-11-11T07:22:49,270 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e6700cbcf8d74d3c855a9541bb71f28b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731309764820 2024-11-11T07:22:49,271 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ada2d31ff54098bb3e2f0ee6a5d66b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731309767039 2024-11-11T07:22:49,272 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd9500c493b44bf82c795faf68fa3b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309767114 2024-11-11T07:22:49,272 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 556da1bf4931452d81dce78dd4f96b5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731309767766 2024-11-11T07:22:49,292 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#C#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:49,294 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/7672f9e4d4eb4604b457b573759bed9c is 50, key is test_row_0/C:col10/1731309768928/Put/seqid=0 2024-11-11T07:22:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:49,304 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:49,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:22:49,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742004_1180 (size=12459) 2024-11-11T07:22:49,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d7c82f1c3864dfdbce0815675adce99_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309768975/Put/seqid=0 2024-11-11T07:22:49,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309829319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309829319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309829319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309829320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:22:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:49,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309829324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742005_1181 (size=14794) 2024-11-11T07:22:49,350 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:49,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d7c82f1c3864dfdbce0815675adce99_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d7c82f1c3864dfdbce0815675adce99_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:49,361 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/db5391e35f0a4984b528b06bc0a9bb69, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:49,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/db5391e35f0a4984b528b06bc0a9bb69 is 175, key is test_row_0/A:col10/1731309768975/Put/seqid=0 2024-11-11T07:22:49,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742006_1182 (size=39749) 2024-11-11T07:22:49,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309829432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309829432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309829432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309829441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309829441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:22:49,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:22:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:22:49,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309829642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309829644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309829644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309829652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309829652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,670 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:49,671 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/3b69df9e3bc3462581da1b9b934f42cd is 175, key is test_row_0/A:col10/1731309768928/Put/seqid=0 2024-11-11T07:22:49,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742007_1183 (size=31447) 2024-11-11T07:22:49,690 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/3b69df9e3bc3462581da1b9b934f42cd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3b69df9e3bc3462581da1b9b934f42cd 2024-11-11T07:22:49,697 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/A of 8315bbc46c6a14c3758164d93d7c2260 into 3b69df9e3bc3462581da1b9b934f42cd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:49,697 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:49,697 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/A, priority=13, startTime=1731309769181; duration=0sec 2024-11-11T07:22:49,697 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:49,697 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:A 2024-11-11T07:22:49,712 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/7672f9e4d4eb4604b457b573759bed9c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/7672f9e4d4eb4604b457b573759bed9c 2024-11-11T07:22:49,721 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/C of 8315bbc46c6a14c3758164d93d7c2260 into 7672f9e4d4eb4604b457b573759bed9c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:49,721 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:49,721 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/C, priority=12, startTime=1731309769181; duration=0sec 2024-11-11T07:22:49,721 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:49,721 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:C 2024-11-11T07:22:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:22:49,776 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/db5391e35f0a4984b528b06bc0a9bb69 2024-11-11T07:22:49,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:22:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/a3dda18c847349c0afd94252ababcf0e is 50, key is test_row_0/B:col10/1731309768975/Put/seqid=0 2024-11-11T07:22:49,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742008_1184 (size=12151) 2024-11-11T07:22:49,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/a3dda18c847349c0afd94252ababcf0e 2024-11-11T07:22:49,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/264114cd40bf40bb9d01b2c9c258bc39 is 50, key is test_row_0/C:col10/1731309768975/Put/seqid=0 2024-11-11T07:22:49,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742009_1185 (size=12151) 2024-11-11T07:22:49,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:22:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:49,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:49,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309829952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309829961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309829962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309829962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309829962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:22:50,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:50,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:22:50,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/264114cd40bf40bb9d01b2c9c258bc39 2024-11-11T07:22:50,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/db5391e35f0a4984b528b06bc0a9bb69 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/db5391e35f0a4984b528b06bc0a9bb69 2024-11-11T07:22:50,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/db5391e35f0a4984b528b06bc0a9bb69, entries=200, sequenceid=171, filesize=38.8 K 2024-11-11T07:22:50,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/a3dda18c847349c0afd94252ababcf0e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a3dda18c847349c0afd94252ababcf0e 2024-11-11T07:22:50,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a3dda18c847349c0afd94252ababcf0e, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:22:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/264114cd40bf40bb9d01b2c9c258bc39 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/264114cd40bf40bb9d01b2c9c258bc39 2024-11-11T07:22:50,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/264114cd40bf40bb9d01b2c9c258bc39, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:22:50,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8315bbc46c6a14c3758164d93d7c2260 in 951ms, sequenceid=171, compaction requested=false 2024-11-11T07:22:50,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:50,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:22:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:50,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:22:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:22:50,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119166a35848ee4d8b8575cc77930f82ca_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309769318/Put/seqid=0 2024-11-11T07:22:50,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742010_1186 (size=12304) 2024-11-11T07:22:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:50,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309830500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309830511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309830511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309830511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309830512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309830622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309830623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309830622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309830624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309830624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:50,701 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119166a35848ee4d8b8575cc77930f82ca_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119166a35848ee4d8b8575cc77930f82ca_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:50,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a4b1120cfb204b78ba0cee8bea3e0af8, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:50,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a4b1120cfb204b78ba0cee8bea3e0af8 is 175, key is test_row_0/A:col10/1731309769318/Put/seqid=0 2024-11-11T07:22:50,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742011_1187 (size=31105) 2024-11-11T07:22:50,734 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a4b1120cfb204b78ba0cee8bea3e0af8 2024-11-11T07:22:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/94fe6d6708f2412ba9cf612449752f63 is 50, key is test_row_0/B:col10/1731309769318/Put/seqid=0 2024-11-11T07:22:50,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742012_1188 (size=12151) 2024-11-11T07:22:50,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309830842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309830842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309830842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309830842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309830842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309831151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309831162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309831162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309831162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309831162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,211 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/94fe6d6708f2412ba9cf612449752f63 2024-11-11T07:22:51,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/1d23bced118f48598a0deb3de694585c is 50, key is test_row_0/C:col10/1731309769318/Put/seqid=0 2024-11-11T07:22:51,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742013_1189 (size=12151) 2024-11-11T07:22:51,275 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/1d23bced118f48598a0deb3de694585c 2024-11-11T07:22:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:22:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a4b1120cfb204b78ba0cee8bea3e0af8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a4b1120cfb204b78ba0cee8bea3e0af8 2024-11-11T07:22:51,292 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a4b1120cfb204b78ba0cee8bea3e0af8, entries=150, sequenceid=186, filesize=30.4 K 2024-11-11T07:22:51,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/94fe6d6708f2412ba9cf612449752f63 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/94fe6d6708f2412ba9cf612449752f63 2024-11-11T07:22:51,298 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/94fe6d6708f2412ba9cf612449752f63, entries=150, sequenceid=186, filesize=11.9 K 2024-11-11T07:22:51,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/1d23bced118f48598a0deb3de694585c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1d23bced118f48598a0deb3de694585c 2024-11-11T07:22:51,307 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1d23bced118f48598a0deb3de694585c, entries=150, sequenceid=186, filesize=11.9 K 2024-11-11T07:22:51,308 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8315bbc46c6a14c3758164d93d7c2260 in 1052ms, sequenceid=186, compaction requested=true 2024-11-11T07:22:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T07:22:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T07:22:51,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T07:22:51,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1470 sec 2024-11-11T07:22:51,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.1780 sec 2024-11-11T07:22:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:51,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:51,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:22:51,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d5acf7fa5f0415b9f8f2e0e568124df_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309771662/Put/seqid=0 2024-11-11T07:22:51,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742014_1190 (size=14794) 2024-11-11T07:22:51,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309831684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309831685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309831686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309831687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309831688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:51,701 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d5acf7fa5f0415b9f8f2e0e568124df_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d5acf7fa5f0415b9f8f2e0e568124df_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:51,702 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/b8d39052d7014a7b91760ce1e6a097db, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:51,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/b8d39052d7014a7b91760ce1e6a097db is 175, key is test_row_0/A:col10/1731309771662/Put/seqid=0 2024-11-11T07:22:51,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742015_1191 (size=39749) 2024-11-11T07:22:51,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309831802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309831802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309831802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309831802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309832012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309832012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309832012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309832014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/b8d39052d7014a7b91760ce1e6a097db 2024-11-11T07:22:52,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/83b201cfc1ae4191a75f8aefdef241d7 is 50, key is test_row_0/B:col10/1731309771662/Put/seqid=0 2024-11-11T07:22:52,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742016_1192 (size=12151) 2024-11-11T07:22:52,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/83b201cfc1ae4191a75f8aefdef241d7 2024-11-11T07:22:52,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/430b796611704d13a6e26e27024738d3 is 50, key is test_row_0/C:col10/1731309771662/Put/seqid=0 2024-11-11T07:22:52,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742017_1193 (size=12151) 2024-11-11T07:22:52,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/430b796611704d13a6e26e27024738d3 2024-11-11T07:22:52,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/b8d39052d7014a7b91760ce1e6a097db as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/b8d39052d7014a7b91760ce1e6a097db 2024-11-11T07:22:52,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/b8d39052d7014a7b91760ce1e6a097db, entries=200, sequenceid=210, filesize=38.8 K 2024-11-11T07:22:52,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/83b201cfc1ae4191a75f8aefdef241d7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/83b201cfc1ae4191a75f8aefdef241d7 2024-11-11T07:22:52,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/83b201cfc1ae4191a75f8aefdef241d7, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T07:22:52,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/430b796611704d13a6e26e27024738d3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/430b796611704d13a6e26e27024738d3 2024-11-11T07:22:52,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/430b796611704d13a6e26e27024738d3, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T07:22:52,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8315bbc46c6a14c3758164d93d7c2260 in 620ms, sequenceid=210, compaction requested=true 2024-11-11T07:22:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:52,285 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:22:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:52,285 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:22:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:22:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:52,286 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:52,287 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/A is initiating minor compaction (all files) 2024-11-11T07:22:52,287 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/A in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:52,287 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3b69df9e3bc3462581da1b9b934f42cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/db5391e35f0a4984b528b06bc0a9bb69, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a4b1120cfb204b78ba0cee8bea3e0af8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/b8d39052d7014a7b91760ce1e6a097db] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=138.7 K 2024-11-11T07:22:52,287 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:52,287 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3b69df9e3bc3462581da1b9b934f42cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/db5391e35f0a4984b528b06bc0a9bb69, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a4b1120cfb204b78ba0cee8bea3e0af8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/b8d39052d7014a7b91760ce1e6a097db] 2024-11-11T07:22:52,287 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b69df9e3bc3462581da1b9b934f42cd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731309767766 2024-11-11T07:22:52,288 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5391e35f0a4984b528b06bc0a9bb69, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731309768971 2024-11-11T07:22:52,288 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:52,288 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/B is initiating minor compaction (all files) 2024-11-11T07:22:52,288 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/B in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:52,288 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e1bccf5fa49c47d3a80d23fb8942bc67, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a3dda18c847349c0afd94252ababcf0e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/94fe6d6708f2412ba9cf612449752f63, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/83b201cfc1ae4191a75f8aefdef241d7] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=47.8 K 2024-11-11T07:22:52,289 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b1120cfb204b78ba0cee8bea3e0af8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731309769310 2024-11-11T07:22:52,289 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e1bccf5fa49c47d3a80d23fb8942bc67, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731309767766 2024-11-11T07:22:52,289 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a3dda18c847349c0afd94252ababcf0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731309768974 2024-11-11T07:22:52,289 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d39052d7014a7b91760ce1e6a097db, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731309770498 2024-11-11T07:22:52,290 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 94fe6d6708f2412ba9cf612449752f63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731309769310 2024-11-11T07:22:52,290 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b201cfc1ae4191a75f8aefdef241d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731309770498 2024-11-11T07:22:52,314 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:52,315 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#B#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:52,316 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/6f36819d83ab47aeb7a9347fc9bed34d is 50, key is test_row_0/B:col10/1731309771662/Put/seqid=0 2024-11-11T07:22:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:52,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:22:52,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:22:52,329 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118ccfabcbe48041cd9d44d3134c56a47b_8315bbc46c6a14c3758164d93d7c2260 store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:52,331 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118ccfabcbe48041cd9d44d3134c56a47b_8315bbc46c6a14c3758164d93d7c2260, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:52,332 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118ccfabcbe48041cd9d44d3134c56a47b_8315bbc46c6a14c3758164d93d7c2260 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:52,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119ec319b846114cd196fbee99800d49bd_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309771680/Put/seqid=0 2024-11-11T07:22:52,398 INFO [master/f334e8b0fbdf:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T07:22:52,398 INFO [master/f334e8b0fbdf:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T07:22:52,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742018_1194 (size=12629) 2024-11-11T07:22:52,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309832390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309832392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309832404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,412 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/6f36819d83ab47aeb7a9347fc9bed34d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/6f36819d83ab47aeb7a9347fc9bed34d 2024-11-11T07:22:52,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742019_1195 (size=4469) 2024-11-11T07:22:52,421 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/B of 8315bbc46c6a14c3758164d93d7c2260 into 6f36819d83ab47aeb7a9347fc9bed34d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:52,421 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:52,421 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/B, priority=12, startTime=1731309772285; duration=0sec 2024-11-11T07:22:52,421 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:22:52,421 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:B 2024-11-11T07:22:52,421 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:22:52,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309832406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,425 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#A#compaction#165 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:52,425 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:22:52,425 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/C is initiating minor compaction (all files) 2024-11-11T07:22:52,425 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/C in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:52,425 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/67ca1f7eba554bb4ac6016ef5fe208a8 is 175, key is test_row_0/A:col10/1731309771662/Put/seqid=0 2024-11-11T07:22:52,426 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/7672f9e4d4eb4604b457b573759bed9c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/264114cd40bf40bb9d01b2c9c258bc39, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1d23bced118f48598a0deb3de694585c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/430b796611704d13a6e26e27024738d3] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=47.8 K 2024-11-11T07:22:52,426 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7672f9e4d4eb4604b457b573759bed9c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731309767766 2024-11-11T07:22:52,426 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 264114cd40bf40bb9d01b2c9c258bc39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731309768974 2024-11-11T07:22:52,427 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d23bced118f48598a0deb3de694585c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731309769310 2024-11-11T07:22:52,427 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 430b796611704d13a6e26e27024738d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731309770498 2024-11-11T07:22:52,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742020_1196 (size=14794) 2024-11-11T07:22:52,439 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:22:52,439 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/b7d72312993c4d7a83064e609920db06 is 50, key is test_row_0/C:col10/1731309771662/Put/seqid=0 2024-11-11T07:22:52,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742021_1197 (size=31583) 2024-11-11T07:22:52,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742022_1198 (size=12595) 2024-11-11T07:22:52,448 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/67ca1f7eba554bb4ac6016ef5fe208a8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/67ca1f7eba554bb4ac6016ef5fe208a8 2024-11-11T07:22:52,460 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/A of 8315bbc46c6a14c3758164d93d7c2260 into 67ca1f7eba554bb4ac6016ef5fe208a8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:52,460 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:52,460 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/A, priority=12, startTime=1731309772285; duration=0sec 2024-11-11T07:22:52,460 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:52,460 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:A 2024-11-11T07:22:52,478 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/b7d72312993c4d7a83064e609920db06 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b7d72312993c4d7a83064e609920db06 2024-11-11T07:22:52,485 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/C of 8315bbc46c6a14c3758164d93d7c2260 into b7d72312993c4d7a83064e609920db06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:22:52,485 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:52,485 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/C, priority=12, startTime=1731309772286; duration=0sec 2024-11-11T07:22:52,485 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:22:52,485 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:C 2024-11-11T07:22:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309832511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309832511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309832511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309832532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309832701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,703 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309832722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309832722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309832722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309832741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:52,835 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:52,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119ec319b846114cd196fbee99800d49bd_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119ec319b846114cd196fbee99800d49bd_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:52,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/76bc7a8c702f40ab8a29d5e3f1a1c88f, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:52,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/76bc7a8c702f40ab8a29d5e3f1a1c88f is 175, key is test_row_0/A:col10/1731309771680/Put/seqid=0 2024-11-11T07:22:52,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742023_1199 (size=39749) 2024-11-11T07:22:53,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309833032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309833032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309833032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309833052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,293 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/76bc7a8c702f40ab8a29d5e3f1a1c88f 2024-11-11T07:22:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:22:53,302 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:53,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T07:22:53,307 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:22:53,308 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:53,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:53,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/cb41fa0572874fae852bf5c74e4b3692 is 50, key is test_row_0/B:col10/1731309771680/Put/seqid=0 2024-11-11T07:22:53,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742024_1200 (size=12151) 2024-11-11T07:22:53,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/cb41fa0572874fae852bf5c74e4b3692 2024-11-11T07:22:53,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/79840279258b4247b5e402fe212b245d is 50, key is test_row_0/C:col10/1731309771680/Put/seqid=0 2024-11-11T07:22:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:22:53,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742025_1201 (size=12151) 2024-11-11T07:22:53,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/79840279258b4247b5e402fe212b245d 2024-11-11T07:22:53,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/76bc7a8c702f40ab8a29d5e3f1a1c88f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/76bc7a8c702f40ab8a29d5e3f1a1c88f 2024-11-11T07:22:53,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/76bc7a8c702f40ab8a29d5e3f1a1c88f, entries=200, sequenceid=224, filesize=38.8 K 2024-11-11T07:22:53,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/cb41fa0572874fae852bf5c74e4b3692 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/cb41fa0572874fae852bf5c74e4b3692 2024-11-11T07:22:53,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/cb41fa0572874fae852bf5c74e4b3692, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T07:22:53,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/79840279258b4247b5e402fe212b245d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/79840279258b4247b5e402fe212b245d 2024-11-11T07:22:53,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/79840279258b4247b5e402fe212b245d, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T07:22:53,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 8315bbc46c6a14c3758164d93d7c2260 in 1130ms, sequenceid=224, compaction requested=false 2024-11-11T07:22:53,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:53,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T07:22:53,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:53,462 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:22:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c5a4624059294b56ab5f87c372425a0a_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309772379/Put/seqid=0 2024-11-11T07:22:53,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742026_1202 (size=12304) 2024-11-11T07:22:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:53,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:53,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309833563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309833564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309833566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309833569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:22:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309833682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309833682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309833692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309833891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309833892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309833902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:22:53,935 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c5a4624059294b56ab5f87c372425a0a_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c5a4624059294b56ab5f87c372425a0a_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/46e82c6b9d774627957aa0a6584b209e, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:53,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/46e82c6b9d774627957aa0a6584b209e is 175, key is test_row_0/A:col10/1731309772379/Put/seqid=0 2024-11-11T07:22:53,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742027_1203 (size=31105) 2024-11-11T07:22:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309834201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309834201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309834222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,367 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/46e82c6b9d774627957aa0a6584b209e 2024-11-11T07:22:54,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/0a4e0a1035884b81a343c16a6c338143 is 50, key is test_row_0/B:col10/1731309772379/Put/seqid=0 2024-11-11T07:22:54,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742028_1204 (size=12151) 2024-11-11T07:22:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:22:54,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309834591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,596 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309834721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,725 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309834731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:54,801 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/0a4e0a1035884b81a343c16a6c338143 2024-11-11T07:22:54,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/1574f0966a0946aa8c1545e8dbcd964e is 50, key is test_row_0/C:col10/1731309772379/Put/seqid=0 2024-11-11T07:22:54,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742029_1205 (size=12151) 2024-11-11T07:22:55,251 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/1574f0966a0946aa8c1545e8dbcd964e 2024-11-11T07:22:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/46e82c6b9d774627957aa0a6584b209e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/46e82c6b9d774627957aa0a6584b209e 2024-11-11T07:22:55,265 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/46e82c6b9d774627957aa0a6584b209e, entries=150, sequenceid=250, filesize=30.4 K 2024-11-11T07:22:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/0a4e0a1035884b81a343c16a6c338143 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/0a4e0a1035884b81a343c16a6c338143 2024-11-11T07:22:55,273 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/0a4e0a1035884b81a343c16a6c338143, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T07:22:55,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/1574f0966a0946aa8c1545e8dbcd964e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1574f0966a0946aa8c1545e8dbcd964e 2024-11-11T07:22:55,280 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1574f0966a0946aa8c1545e8dbcd964e, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T07:22:55,281 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8315bbc46c6a14c3758164d93d7c2260 in 1819ms, sequenceid=250, compaction requested=true 2024-11-11T07:22:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:55,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T07:22:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T07:22:55,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T07:22:55,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-11-11T07:22:55,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.9890 sec 2024-11-11T07:22:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:22:55,452 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:55,452 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T07:22:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:22:55,455 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:55,456 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:55,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:22:55,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T07:22:55,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:55,609 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:22:55,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b935f867b38a4748bb5760561e139610_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309773563/Put/seqid=0 2024-11-11T07:22:55,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742030_1206 (size=12454) 2024-11-11T07:22:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:22:55,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309835773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:55,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309835774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309835775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:55,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309835882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:55,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:55,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309835882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:55,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:55,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309835891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:56,027 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b935f867b38a4748bb5760561e139610_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b935f867b38a4748bb5760561e139610_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/f8d509fca56942a0b868c518a2a92211, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/f8d509fca56942a0b868c518a2a92211 is 175, key is test_row_0/A:col10/1731309773563/Put/seqid=0 2024-11-11T07:22:56,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742031_1207 (size=31255) 2024-11-11T07:22:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:22:56,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309836091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309836091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309836101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309836402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309836402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309836412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,433 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/f8d509fca56942a0b868c518a2a92211 2024-11-11T07:22:56,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/751e971499544fd6ba0dfcd75e526409 is 50, key is test_row_0/B:col10/1731309773563/Put/seqid=0 2024-11-11T07:22:56,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742032_1208 (size=12301) 2024-11-11T07:22:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:22:56,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309836612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,614 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,846 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/751e971499544fd6ba0dfcd75e526409 2024-11-11T07:22:56,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/e93883a39e184ba59b905ab8ec7cc957 is 50, key is test_row_0/C:col10/1731309773563/Put/seqid=0 2024-11-11T07:22:56,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742033_1209 (size=12301) 2024-11-11T07:22:56,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309836911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309836911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:56,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309836921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:57,263 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/e93883a39e184ba59b905ab8ec7cc957 2024-11-11T07:22:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/f8d509fca56942a0b868c518a2a92211 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/f8d509fca56942a0b868c518a2a92211 2024-11-11T07:22:57,273 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/f8d509fca56942a0b868c518a2a92211, entries=150, sequenceid=263, filesize=30.5 K 2024-11-11T07:22:57,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/751e971499544fd6ba0dfcd75e526409 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/751e971499544fd6ba0dfcd75e526409 2024-11-11T07:22:57,279 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/751e971499544fd6ba0dfcd75e526409, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T07:22:57,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/e93883a39e184ba59b905ab8ec7cc957 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e93883a39e184ba59b905ab8ec7cc957 2024-11-11T07:22:57,284 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e93883a39e184ba59b905ab8ec7cc957, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T07:22:57,286 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8315bbc46c6a14c3758164d93d7c2260 in 1677ms, sequenceid=263, compaction requested=true 2024-11-11T07:22:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T07:22:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T07:22:57,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T07:22:57,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-11T07:22:57,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8370 sec 2024-11-11T07:22:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:22:57,602 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:57,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T07:22:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T07:22:57,604 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:57,605 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:57,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T07:22:57,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T07:22:57,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:57,758 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:22:57,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bf758028df9b4021aa45c3650764bcaa_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309775766/Put/seqid=0 2024-11-11T07:22:57,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742034_1210 (size=12454) 2024-11-11T07:22:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T07:22:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:57,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:22:57,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309837940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:57,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:57,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:57,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:57,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309837941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:57,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309837941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:57,951 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309838051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309838062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:22:58,175 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bf758028df9b4021aa45c3650764bcaa_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf758028df9b4021aa45c3650764bcaa_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:58,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/5cb2ea27cffc414c88fe87a559984d53, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:22:58,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/5cb2ea27cffc414c88fe87a559984d53 is 175, key is test_row_0/A:col10/1731309775766/Put/seqid=0 2024-11-11T07:22:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742035_1211 (size=31255) 2024-11-11T07:22:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T07:22:58,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309838261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309838272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309838571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,581 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/5cb2ea27cffc414c88fe87a559984d53 2024-11-11T07:22:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/7453f8946cad4a6383fcb13d7163d621 is 50, key is test_row_0/B:col10/1731309775766/Put/seqid=0 2024-11-11T07:22:58,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742036_1212 (size=12301) 2024-11-11T07:22:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309838591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T07:22:58,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51504 deadline: 1731309838772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:58,775 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:22:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:58,994 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/7453f8946cad4a6383fcb13d7163d621 2024-11-11T07:22:59,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/6e99be0190494f08b6c917776a276d4c is 50, key is test_row_0/C:col10/1731309775766/Put/seqid=0 2024-11-11T07:22:59,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742037_1213 (size=12301) 2024-11-11T07:22:59,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309839081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:59,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:22:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309839101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:22:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:22:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:22:59,411 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/6e99be0190494f08b6c917776a276d4c 2024-11-11T07:22:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/5cb2ea27cffc414c88fe87a559984d53 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/5cb2ea27cffc414c88fe87a559984d53 2024-11-11T07:22:59,420 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/5cb2ea27cffc414c88fe87a559984d53, entries=150, sequenceid=286, filesize=30.5 K 2024-11-11T07:22:59,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/7453f8946cad4a6383fcb13d7163d621 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7453f8946cad4a6383fcb13d7163d621 2024-11-11T07:22:59,426 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7453f8946cad4a6383fcb13d7163d621, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T07:22:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/6e99be0190494f08b6c917776a276d4c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/6e99be0190494f08b6c917776a276d4c 2024-11-11T07:22:59,433 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/6e99be0190494f08b6c917776a276d4c, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T07:22:59,434 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8315bbc46c6a14c3758164d93d7c2260 in 1676ms, sequenceid=286, compaction requested=true 2024-11-11T07:22:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:22:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T07:22:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T07:22:59,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T07:22:59,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-11T07:22:59,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8370 sec 2024-11-11T07:22:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T07:22:59,752 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:22:59,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:22:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-11T07:22:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:22:59,754 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:22:59,755 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:22:59,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:22:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:22:59,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T07:22:59,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:22:59,908 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:22:59,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c72741432f0c40319a01ea29faa72e4e_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309777938/Put/seqid=0 2024-11-11T07:22:59,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742038_1214 (size=12454) 2024-11-11T07:22:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:22:59,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. as already flushing 2024-11-11T07:23:00,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309840063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:23:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309840101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:00,103 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:00,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309840111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:00,117 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:00,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309840172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:00,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:00,332 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c72741432f0c40319a01ea29faa72e4e_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c72741432f0c40319a01ea29faa72e4e_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/cd1d0225d5304efebd92913f768e4eb9, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:00,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/cd1d0225d5304efebd92913f768e4eb9 is 175, key is test_row_0/A:col10/1731309777938/Put/seqid=0 2024-11-11T07:23:00,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742039_1215 (size=31255) 2024-11-11T07:23:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:23:00,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309840381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51482 deadline: 1731309840631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:00,634 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:00,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309840692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:00,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:00,738 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/cd1d0225d5304efebd92913f768e4eb9 2024-11-11T07:23:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e9577f04ab48434da02009a1eb4e8d05 is 50, key is test_row_0/B:col10/1731309777938/Put/seqid=0 2024-11-11T07:23:00,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742040_1216 (size=12301) 2024-11-11T07:23:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:23:01,153 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e9577f04ab48434da02009a1eb4e8d05 2024-11-11T07:23:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/cd430f9a9d554a37b9fab4f53e8562f4 is 50, key is test_row_0/C:col10/1731309777938/Put/seqid=0 2024-11-11T07:23:01,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742041_1217 (size=12301) 2024-11-11T07:23:01,166 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/cd430f9a9d554a37b9fab4f53e8562f4 2024-11-11T07:23:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/cd1d0225d5304efebd92913f768e4eb9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/cd1d0225d5304efebd92913f768e4eb9 2024-11-11T07:23:01,175 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/cd1d0225d5304efebd92913f768e4eb9, entries=150, sequenceid=299, filesize=30.5 K 2024-11-11T07:23:01,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/e9577f04ab48434da02009a1eb4e8d05 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e9577f04ab48434da02009a1eb4e8d05 2024-11-11T07:23:01,180 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e9577f04ab48434da02009a1eb4e8d05, entries=150, sequenceid=299, filesize=12.0 K 2024-11-11T07:23:01,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/cd430f9a9d554a37b9fab4f53e8562f4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/cd430f9a9d554a37b9fab4f53e8562f4 2024-11-11T07:23:01,185 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/cd430f9a9d554a37b9fab4f53e8562f4, entries=150, sequenceid=299, filesize=12.0 K 2024-11-11T07:23:01,186 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8315bbc46c6a14c3758164d93d7c2260 in 1277ms, sequenceid=299, compaction requested=true 2024-11-11T07:23:01,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:23:01,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:01,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-11T07:23:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-11T07:23:01,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T07:23:01,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-11-11T07:23:01,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4360 sec 2024-11-11T07:23:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:01,207 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:01,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:23:01,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112965223ab0a64980a6fc6fbf552b937f_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309780055/Put/seqid=0 2024-11-11T07:23:01,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742042_1218 (size=14994) 2024-11-11T07:23:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309841249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:01,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309841361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:01,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309841572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:01,628 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:01,633 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112965223ab0a64980a6fc6fbf552b937f_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112965223ab0a64980a6fc6fbf552b937f_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:01,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/57bc6c01dde74930ba11f82eabd05897, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:01,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/57bc6c01dde74930ba11f82eabd05897 is 175, key is test_row_0/A:col10/1731309780055/Put/seqid=0 2024-11-11T07:23:01,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742043_1219 (size=39949) 2024-11-11T07:23:01,663 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-11T07:23:01,663 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:01,663 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,663 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:01,664 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-11T07:23:01,664 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:01,664 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,664 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:01,668 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-11T07:23:01,668 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:01,668 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,668 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:01,673 INFO [Thread-701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-701. 2024-11-11T07:23:01,673 DEBUG [Thread-701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:01,673 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,673 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:01,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309841882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:01,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:01,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:23:01,912 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:02,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/57bc6c01dde74930ba11f82eabd05897 2024-11-11T07:23:02,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/699d6b0b73df46929019171c66acbcf2 is 50, key is test_row_0/B:col10/1731309780055/Put/seqid=0 2024-11-11T07:23:02,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742044_1220 (size=12301) 2024-11-11T07:23:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51470 deadline: 1731309842112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:02,113 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:02,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51492 deadline: 1731309842131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:02,132 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 8315bbc46c6a14c3758164d93d7c2260 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:02,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51552 deadline: 1731309842391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:02,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/699d6b0b73df46929019171c66acbcf2 2024-11-11T07:23:02,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/2493572ebedf43208b7413e1a17d4f5a is 50, key is test_row_0/C:col10/1731309780055/Put/seqid=0 2024-11-11T07:23:02,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742045_1221 (size=12301) 2024-11-11T07:23:02,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/2493572ebedf43208b7413e1a17d4f5a 2024-11-11T07:23:02,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/57bc6c01dde74930ba11f82eabd05897 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/57bc6c01dde74930ba11f82eabd05897 2024-11-11T07:23:02,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/57bc6c01dde74930ba11f82eabd05897, entries=200, sequenceid=323, filesize=39.0 K 2024-11-11T07:23:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/699d6b0b73df46929019171c66acbcf2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/699d6b0b73df46929019171c66acbcf2 2024-11-11T07:23:02,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/699d6b0b73df46929019171c66acbcf2, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T07:23:02,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/2493572ebedf43208b7413e1a17d4f5a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/2493572ebedf43208b7413e1a17d4f5a 2024-11-11T07:23:02,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/2493572ebedf43208b7413e1a17d4f5a, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T07:23:02,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8315bbc46c6a14c3758164d93d7c2260 in 1674ms, sequenceid=323, compaction requested=true 2024-11-11T07:23:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:23:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:02,881 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-11T07:23:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8315bbc46c6a14c3758164d93d7c2260:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:02,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:02,881 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-11T07:23:02,883 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 236151 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-11T07:23:02,883 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86135 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-11T07:23:02,883 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/A is initiating minor compaction (all files) 2024-11-11T07:23:02,883 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/B is initiating minor compaction (all files) 2024-11-11T07:23:02,883 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/A in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:02,883 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/B in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:02,883 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/67ca1f7eba554bb4ac6016ef5fe208a8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/76bc7a8c702f40ab8a29d5e3f1a1c88f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/46e82c6b9d774627957aa0a6584b209e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/f8d509fca56942a0b868c518a2a92211, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/5cb2ea27cffc414c88fe87a559984d53, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/cd1d0225d5304efebd92913f768e4eb9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/57bc6c01dde74930ba11f82eabd05897] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=230.6 K 2024-11-11T07:23:02,883 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/6f36819d83ab47aeb7a9347fc9bed34d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/cb41fa0572874fae852bf5c74e4b3692, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/0a4e0a1035884b81a343c16a6c338143, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/751e971499544fd6ba0dfcd75e526409, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7453f8946cad4a6383fcb13d7163d621, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e9577f04ab48434da02009a1eb4e8d05, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/699d6b0b73df46929019171c66acbcf2] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=84.1 K 2024-11-11T07:23:02,883 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:02,883 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/67ca1f7eba554bb4ac6016ef5fe208a8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/76bc7a8c702f40ab8a29d5e3f1a1c88f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/46e82c6b9d774627957aa0a6584b209e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/f8d509fca56942a0b868c518a2a92211, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/5cb2ea27cffc414c88fe87a559984d53, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/cd1d0225d5304efebd92913f768e4eb9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/57bc6c01dde74930ba11f82eabd05897] 2024-11-11T07:23:02,884 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f36819d83ab47aeb7a9347fc9bed34d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731309770498 2024-11-11T07:23:02,884 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ca1f7eba554bb4ac6016ef5fe208a8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731309770498 2024-11-11T07:23:02,884 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cb41fa0572874fae852bf5c74e4b3692, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731309771680 2024-11-11T07:23:02,884 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76bc7a8c702f40ab8a29d5e3f1a1c88f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731309771680 2024-11-11T07:23:02,884 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4e0a1035884b81a343c16a6c338143, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731309772379 2024-11-11T07:23:02,885 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e82c6b9d774627957aa0a6584b209e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731309772379 2024-11-11T07:23:02,885 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 751e971499544fd6ba0dfcd75e526409, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309773557 2024-11-11T07:23:02,885 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8d509fca56942a0b868c518a2a92211, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309773557 2024-11-11T07:23:02,885 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7453f8946cad4a6383fcb13d7163d621, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731309775766 2024-11-11T07:23:02,885 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb2ea27cffc414c88fe87a559984d53, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731309775766 2024-11-11T07:23:02,885 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e9577f04ab48434da02009a1eb4e8d05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731309777935 2024-11-11T07:23:02,885 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd1d0225d5304efebd92913f768e4eb9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731309777935 2024-11-11T07:23:02,886 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 699d6b0b73df46929019171c66acbcf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731309780045 2024-11-11T07:23:02,886 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57bc6c01dde74930ba11f82eabd05897, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731309780045 2024-11-11T07:23:02,899 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:02,901 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c02bd47a2039423f895358fa407fd842_8315bbc46c6a14c3758164d93d7c2260 store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:02,902 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#B#compaction#185 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:02,903 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/08ebeba424214b068b335e5284783485 is 50, key is test_row_0/B:col10/1731309780055/Put/seqid=0 2024-11-11T07:23:02,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742046_1222 (size=13017) 2024-11-11T07:23:02,918 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/08ebeba424214b068b335e5284783485 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/08ebeba424214b068b335e5284783485 2024-11-11T07:23:02,924 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/B of 8315bbc46c6a14c3758164d93d7c2260 into 08ebeba424214b068b335e5284783485(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:02,925 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:23:02,925 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/B, priority=9, startTime=1731309782881; duration=0sec 2024-11-11T07:23:02,925 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:02,925 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:B 2024-11-11T07:23:02,925 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-11T07:23:02,927 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86101 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-11T07:23:02,927 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 8315bbc46c6a14c3758164d93d7c2260/C is initiating minor compaction (all files) 2024-11-11T07:23:02,927 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8315bbc46c6a14c3758164d93d7c2260/C in TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:02,927 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b7d72312993c4d7a83064e609920db06, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/79840279258b4247b5e402fe212b245d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1574f0966a0946aa8c1545e8dbcd964e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e93883a39e184ba59b905ab8ec7cc957, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/6e99be0190494f08b6c917776a276d4c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/cd430f9a9d554a37b9fab4f53e8562f4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/2493572ebedf43208b7413e1a17d4f5a] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp, totalSize=84.1 K 2024-11-11T07:23:02,928 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d72312993c4d7a83064e609920db06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731309770498 2024-11-11T07:23:02,928 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 79840279258b4247b5e402fe212b245d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731309771680 2024-11-11T07:23:02,928 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1574f0966a0946aa8c1545e8dbcd964e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731309772379 2024-11-11T07:23:02,929 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e93883a39e184ba59b905ab8ec7cc957, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309773557 2024-11-11T07:23:02,929 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e99be0190494f08b6c917776a276d4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731309775766 2024-11-11T07:23:02,930 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cd430f9a9d554a37b9fab4f53e8562f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731309777935 2024-11-11T07:23:02,930 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2493572ebedf43208b7413e1a17d4f5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731309780045 2024-11-11T07:23:02,941 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c02bd47a2039423f895358fa407fd842_8315bbc46c6a14c3758164d93d7c2260, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:02,941 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c02bd47a2039423f895358fa407fd842_8315bbc46c6a14c3758164d93d7c2260 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:02,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742047_1223 (size=4469) 2024-11-11T07:23:02,960 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#C#compaction#187 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:02,961 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/e47745aa900544e9b7067364f01e2524 is 50, key is test_row_0/C:col10/1731309780055/Put/seqid=0 2024-11-11T07:23:02,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742048_1224 (size=12983) 2024-11-11T07:23:03,347 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8315bbc46c6a14c3758164d93d7c2260#A#compaction#186 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:03,347 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/e8fe63cadd0440c09b2bf1b60255c511 is 175, key is test_row_0/A:col10/1731309780055/Put/seqid=0 2024-11-11T07:23:03,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742049_1225 (size=31971) 2024-11-11T07:23:03,370 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/e47745aa900544e9b7067364f01e2524 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e47745aa900544e9b7067364f01e2524 2024-11-11T07:23:03,375 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/C of 8315bbc46c6a14c3758164d93d7c2260 into e47745aa900544e9b7067364f01e2524(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:03,375 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:23:03,375 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/C, priority=9, startTime=1731309782881; duration=0sec 2024-11-11T07:23:03,375 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:03,375 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:C 2024-11-11T07:23:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:03,403 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:03,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:23:03,403 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-11T07:23:03,403 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:03,403 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:03,403 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:03,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:03,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114cd5bc2477ca4c8cb104e80a68e07a66_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309783402/Put/seqid=0 2024-11-11T07:23:03,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742050_1226 (size=12454) 2024-11-11T07:23:03,756 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/e8fe63cadd0440c09b2bf1b60255c511 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/e8fe63cadd0440c09b2bf1b60255c511 2024-11-11T07:23:03,761 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 8315bbc46c6a14c3758164d93d7c2260/A of 8315bbc46c6a14c3758164d93d7c2260 into e8fe63cadd0440c09b2bf1b60255c511(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:03,761 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:23:03,761 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., storeName=8315bbc46c6a14c3758164d93d7c2260/A, priority=9, startTime=1731309782881; duration=0sec 2024-11-11T07:23:03,761 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:03,761 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8315bbc46c6a14c3758164d93d7c2260:A 2024-11-11T07:23:03,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:03,817 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114cd5bc2477ca4c8cb104e80a68e07a66_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114cd5bc2477ca4c8cb104e80a68e07a66_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:03,817 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/9bb4b9cec5544b28b47bd44957065501, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:03,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/9bb4b9cec5544b28b47bd44957065501 is 175, key is test_row_0/A:col10/1731309783402/Put/seqid=0 2024-11-11T07:23:03,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742051_1227 (size=31255) 2024-11-11T07:23:04,222 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/9bb4b9cec5544b28b47bd44957065501 2024-11-11T07:23:04,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/434e6119d1924a98ad78f31b2fe7292f is 50, key is test_row_0/B:col10/1731309783402/Put/seqid=0 2024-11-11T07:23:04,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742052_1228 (size=12301) 2024-11-11T07:23:04,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/434e6119d1924a98ad78f31b2fe7292f 2024-11-11T07:23:04,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/3a0f13e9d03f458f81160ff3b72691bb is 50, key is test_row_0/C:col10/1731309783402/Put/seqid=0 2024-11-11T07:23:04,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742053_1229 (size=12301) 2024-11-11T07:23:05,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/3a0f13e9d03f458f81160ff3b72691bb 2024-11-11T07:23:05,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/9bb4b9cec5544b28b47bd44957065501 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/9bb4b9cec5544b28b47bd44957065501 2024-11-11T07:23:05,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/9bb4b9cec5544b28b47bd44957065501, entries=150, sequenceid=338, filesize=30.5 K 2024-11-11T07:23:05,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/434e6119d1924a98ad78f31b2fe7292f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/434e6119d1924a98ad78f31b2fe7292f 2024-11-11T07:23:05,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/434e6119d1924a98ad78f31b2fe7292f, entries=150, sequenceid=338, filesize=12.0 K 2024-11-11T07:23:05,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/3a0f13e9d03f458f81160ff3b72691bb as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/3a0f13e9d03f458f81160ff3b72691bb 2024-11-11T07:23:05,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/3a0f13e9d03f458f81160ff3b72691bb, entries=150, sequenceid=338, filesize=12.0 K 2024-11-11T07:23:05,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 8315bbc46c6a14c3758164d93d7c2260 in 1657ms, sequenceid=338, compaction requested=false 2024-11-11T07:23:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8315bbc46c6a14c3758164d93d7c2260: 2024-11-11T07:23:06,143 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-11T07:23:06,143 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-11T07:23:06,143 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:06,143 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:06,143 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:06,143 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:06,144 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:06,144 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:06,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:06,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:06,528 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:23:08,793 INFO [Thread-692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-692. 2024-11-11T07:23:08,793 DEBUG [Thread-692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:08,793 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:08,793 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:08,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:10,693 INFO [Thread-690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-690. 2024-11-11T07:23:10,694 DEBUG [Thread-690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:10,694 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:10,694 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:10,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2727 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2740 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1228 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3684 rows 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1218 2024-11-11T07:23:10,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3652 rows 2024-11-11T07:23:10,694 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:23:10,694 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:23:10,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:10,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:10,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:10,695 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:23:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T07:23:10,698 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309790698"}]},"ts":"1731309790698"} 2024-11-11T07:23:10,699 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:23:10,699 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:23:10,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:23:10,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, UNASSIGN}] 2024-11-11T07:23:10,702 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, UNASSIGN 2024-11-11T07:23:10,703 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:10,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, UNASSIGN because future has completed 2024-11-11T07:23:10,704 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:23:10,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:23:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T07:23:10,856 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:10,856 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:23:10,856 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 8315bbc46c6a14c3758164d93d7c2260, disabling compactions & flushes 2024-11-11T07:23:10,857 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:10,857 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:10,857 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. after waiting 0 ms 2024-11-11T07:23:10,857 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:10,857 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 8315bbc46c6a14c3758164d93d7c2260 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T07:23:10,863 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111159dddae4859544748fdbca6321a32051_8315bbc46c6a14c3758164d93d7c2260 is 50, key is test_row_0/A:col10/1731309788792/Put/seqid=0 2024-11-11T07:23:10,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742054_1230 (size=12454) 2024-11-11T07:23:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T07:23:11,267 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:11,271 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111159dddae4859544748fdbca6321a32051_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111159dddae4859544748fdbca6321a32051_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:11,272 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a869f8f47d96406180fc361b6c2a3230, store: [table=TestAcidGuarantees family=A region=8315bbc46c6a14c3758164d93d7c2260] 2024-11-11T07:23:11,273 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a869f8f47d96406180fc361b6c2a3230 is 175, key is test_row_0/A:col10/1731309788792/Put/seqid=0 2024-11-11T07:23:11,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742055_1231 (size=31255) 2024-11-11T07:23:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T07:23:11,677 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a869f8f47d96406180fc361b6c2a3230 2024-11-11T07:23:11,683 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/9171fc83c0f944819ed84c2e010f4f7c is 50, key is test_row_0/B:col10/1731309788792/Put/seqid=0 2024-11-11T07:23:11,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742056_1232 (size=12301) 2024-11-11T07:23:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T07:23:12,087 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/9171fc83c0f944819ed84c2e010f4f7c 2024-11-11T07:23:12,094 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/67c1239312ae4573be94a2549200980c is 50, key is test_row_0/C:col10/1731309788792/Put/seqid=0 2024-11-11T07:23:12,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742057_1233 (size=12301) 2024-11-11T07:23:12,498 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/67c1239312ae4573be94a2549200980c 2024-11-11T07:23:12,502 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/A/a869f8f47d96406180fc361b6c2a3230 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a869f8f47d96406180fc361b6c2a3230 2024-11-11T07:23:12,505 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a869f8f47d96406180fc361b6c2a3230, entries=150, sequenceid=346, filesize=30.5 K 2024-11-11T07:23:12,506 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/B/9171fc83c0f944819ed84c2e010f4f7c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/9171fc83c0f944819ed84c2e010f4f7c 2024-11-11T07:23:12,509 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/9171fc83c0f944819ed84c2e010f4f7c, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T07:23:12,510 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/.tmp/C/67c1239312ae4573be94a2549200980c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67c1239312ae4573be94a2549200980c 2024-11-11T07:23:12,513 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67c1239312ae4573be94a2549200980c, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T07:23:12,514 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8315bbc46c6a14c3758164d93d7c2260 in 1657ms, sequenceid=346, compaction requested=true 2024-11-11T07:23:12,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/03ecf29b26494def89d26ba775143e39, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3ae3b918af1447df8f0a5490e352fda2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/573b1a2190c148698c4b93820d6d4e8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/0f33a9b8b0ae4469875e3b6a724b6e56, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/d1caaea6126f4623b94b2ed57521ffd6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ac97c97d2aca4da0a7617ae38bc7725a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/c1b18000a053489c9ca7d64f30cc3308, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ab2e328b35fa45f992be1e8937f544aa, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/2a9d7b35073946a4b7450ca790eb11ba, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/eebc4565f6b840f491c6c78c5882b4f6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3b69df9e3bc3462581da1b9b934f42cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/db5391e35f0a4984b528b06bc0a9bb69, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a4b1120cfb204b78ba0cee8bea3e0af8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/b8d39052d7014a7b91760ce1e6a097db, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/67ca1f7eba554bb4ac6016ef5fe208a8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/76bc7a8c702f40ab8a29d5e3f1a1c88f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/46e82c6b9d774627957aa0a6584b209e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/f8d509fca56942a0b868c518a2a92211, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/5cb2ea27cffc414c88fe87a559984d53, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/cd1d0225d5304efebd92913f768e4eb9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/57bc6c01dde74930ba11f82eabd05897] to archive 2024-11-11T07:23:12,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:23:12,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/03ecf29b26494def89d26ba775143e39 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/03ecf29b26494def89d26ba775143e39 2024-11-11T07:23:12,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3ae3b918af1447df8f0a5490e352fda2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3ae3b918af1447df8f0a5490e352fda2 2024-11-11T07:23:12,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/573b1a2190c148698c4b93820d6d4e8c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/573b1a2190c148698c4b93820d6d4e8c 2024-11-11T07:23:12,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/0f33a9b8b0ae4469875e3b6a724b6e56 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/0f33a9b8b0ae4469875e3b6a724b6e56 2024-11-11T07:23:12,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/d1caaea6126f4623b94b2ed57521ffd6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/d1caaea6126f4623b94b2ed57521ffd6 2024-11-11T07:23:12,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ac97c97d2aca4da0a7617ae38bc7725a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ac97c97d2aca4da0a7617ae38bc7725a 2024-11-11T07:23:12,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/c1b18000a053489c9ca7d64f30cc3308 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/c1b18000a053489c9ca7d64f30cc3308 2024-11-11T07:23:12,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ab2e328b35fa45f992be1e8937f544aa to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/ab2e328b35fa45f992be1e8937f544aa 2024-11-11T07:23:12,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/2a9d7b35073946a4b7450ca790eb11ba to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/2a9d7b35073946a4b7450ca790eb11ba 2024-11-11T07:23:12,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/eebc4565f6b840f491c6c78c5882b4f6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/eebc4565f6b840f491c6c78c5882b4f6 2024-11-11T07:23:12,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3b69df9e3bc3462581da1b9b934f42cd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/3b69df9e3bc3462581da1b9b934f42cd 2024-11-11T07:23:12,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/db5391e35f0a4984b528b06bc0a9bb69 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/db5391e35f0a4984b528b06bc0a9bb69 2024-11-11T07:23:12,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a4b1120cfb204b78ba0cee8bea3e0af8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a4b1120cfb204b78ba0cee8bea3e0af8 2024-11-11T07:23:12,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/b8d39052d7014a7b91760ce1e6a097db to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/b8d39052d7014a7b91760ce1e6a097db 2024-11-11T07:23:12,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/67ca1f7eba554bb4ac6016ef5fe208a8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/67ca1f7eba554bb4ac6016ef5fe208a8 2024-11-11T07:23:12,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/76bc7a8c702f40ab8a29d5e3f1a1c88f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/76bc7a8c702f40ab8a29d5e3f1a1c88f 2024-11-11T07:23:12,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/46e82c6b9d774627957aa0a6584b209e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/46e82c6b9d774627957aa0a6584b209e 2024-11-11T07:23:12,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/f8d509fca56942a0b868c518a2a92211 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/f8d509fca56942a0b868c518a2a92211 2024-11-11T07:23:12,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/5cb2ea27cffc414c88fe87a559984d53 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/5cb2ea27cffc414c88fe87a559984d53 2024-11-11T07:23:12,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/cd1d0225d5304efebd92913f768e4eb9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/cd1d0225d5304efebd92913f768e4eb9 2024-11-11T07:23:12,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/57bc6c01dde74930ba11f82eabd05897 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/57bc6c01dde74930ba11f82eabd05897 2024-11-11T07:23:12,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e6e7794a5e454f609dce28ce835f3b97, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/06e5dcc5819443e0a1e3899cebc434f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ecf3aaaa9f79479aad3ac63d30b2d83c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7d2334a7be1542129da6aac01bb3a2ad, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/32000795dba64d7dad91e5ca0405138d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a923213afa154c5e8b380e070b7cfbf8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/17618920ef914f6ba4284cdfa6ce93b5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/adf58054400c42cebbdaf38d7dad7141, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/057ec8ba381c49d481c3bb4755a9453f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e1bccf5fa49c47d3a80d23fb8942bc67, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ef07a3b5712e43ebb0513ca6e640ffe9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a3dda18c847349c0afd94252ababcf0e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/94fe6d6708f2412ba9cf612449752f63, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/6f36819d83ab47aeb7a9347fc9bed34d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/83b201cfc1ae4191a75f8aefdef241d7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/cb41fa0572874fae852bf5c74e4b3692, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/0a4e0a1035884b81a343c16a6c338143, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/751e971499544fd6ba0dfcd75e526409, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7453f8946cad4a6383fcb13d7163d621, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e9577f04ab48434da02009a1eb4e8d05, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/699d6b0b73df46929019171c66acbcf2] to archive 2024-11-11T07:23:12,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:23:12,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e6e7794a5e454f609dce28ce835f3b97 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e6e7794a5e454f609dce28ce835f3b97 2024-11-11T07:23:12,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/06e5dcc5819443e0a1e3899cebc434f8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/06e5dcc5819443e0a1e3899cebc434f8 2024-11-11T07:23:12,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ecf3aaaa9f79479aad3ac63d30b2d83c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ecf3aaaa9f79479aad3ac63d30b2d83c 2024-11-11T07:23:12,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7d2334a7be1542129da6aac01bb3a2ad to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7d2334a7be1542129da6aac01bb3a2ad 2024-11-11T07:23:12,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/32000795dba64d7dad91e5ca0405138d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/32000795dba64d7dad91e5ca0405138d 2024-11-11T07:23:12,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a923213afa154c5e8b380e070b7cfbf8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a923213afa154c5e8b380e070b7cfbf8 2024-11-11T07:23:12,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/17618920ef914f6ba4284cdfa6ce93b5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/17618920ef914f6ba4284cdfa6ce93b5 2024-11-11T07:23:12,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/adf58054400c42cebbdaf38d7dad7141 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/adf58054400c42cebbdaf38d7dad7141 2024-11-11T07:23:12,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/057ec8ba381c49d481c3bb4755a9453f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/057ec8ba381c49d481c3bb4755a9453f 2024-11-11T07:23:12,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e1bccf5fa49c47d3a80d23fb8942bc67 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e1bccf5fa49c47d3a80d23fb8942bc67 2024-11-11T07:23:12,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ef07a3b5712e43ebb0513ca6e640ffe9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/ef07a3b5712e43ebb0513ca6e640ffe9 2024-11-11T07:23:12,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a3dda18c847349c0afd94252ababcf0e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/a3dda18c847349c0afd94252ababcf0e 2024-11-11T07:23:12,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/94fe6d6708f2412ba9cf612449752f63 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/94fe6d6708f2412ba9cf612449752f63 2024-11-11T07:23:12,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/6f36819d83ab47aeb7a9347fc9bed34d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/6f36819d83ab47aeb7a9347fc9bed34d 2024-11-11T07:23:12,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/83b201cfc1ae4191a75f8aefdef241d7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/83b201cfc1ae4191a75f8aefdef241d7 2024-11-11T07:23:12,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/cb41fa0572874fae852bf5c74e4b3692 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/cb41fa0572874fae852bf5c74e4b3692 2024-11-11T07:23:12,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/0a4e0a1035884b81a343c16a6c338143 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/0a4e0a1035884b81a343c16a6c338143 2024-11-11T07:23:12,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/751e971499544fd6ba0dfcd75e526409 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/751e971499544fd6ba0dfcd75e526409 2024-11-11T07:23:12,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7453f8946cad4a6383fcb13d7163d621 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/7453f8946cad4a6383fcb13d7163d621 2024-11-11T07:23:12,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e9577f04ab48434da02009a1eb4e8d05 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/e9577f04ab48434da02009a1eb4e8d05 2024-11-11T07:23:12,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/699d6b0b73df46929019171c66acbcf2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/699d6b0b73df46929019171c66acbcf2 2024-11-11T07:23:12,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/ff0d548b3cf648a698f46665295fc7bd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/c63520948d69458da12bbabbe2a93633, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/705b9e3e30a343848be83505d34676bc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/d65ee4b1e9974d9bbce71d54380d95d6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e6700cbcf8d74d3c855a9541bb71f28b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67ada2d31ff54098bb3e2f0ee6a5d66b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/5fd9500c493b44bf82c795faf68fa3b7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/7672f9e4d4eb4604b457b573759bed9c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/556da1bf4931452d81dce78dd4f96b5c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/264114cd40bf40bb9d01b2c9c258bc39, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1d23bced118f48598a0deb3de694585c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b7d72312993c4d7a83064e609920db06, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/430b796611704d13a6e26e27024738d3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/79840279258b4247b5e402fe212b245d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1574f0966a0946aa8c1545e8dbcd964e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e93883a39e184ba59b905ab8ec7cc957, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/6e99be0190494f08b6c917776a276d4c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/cd430f9a9d554a37b9fab4f53e8562f4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/2493572ebedf43208b7413e1a17d4f5a] to archive 2024-11-11T07:23:12,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:23:12,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/ff0d548b3cf648a698f46665295fc7bd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/ff0d548b3cf648a698f46665295fc7bd 2024-11-11T07:23:12,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/c63520948d69458da12bbabbe2a93633 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/c63520948d69458da12bbabbe2a93633 2024-11-11T07:23:12,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/705b9e3e30a343848be83505d34676bc to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/705b9e3e30a343848be83505d34676bc 2024-11-11T07:23:12,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/d65ee4b1e9974d9bbce71d54380d95d6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/d65ee4b1e9974d9bbce71d54380d95d6 2024-11-11T07:23:12,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e6700cbcf8d74d3c855a9541bb71f28b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e6700cbcf8d74d3c855a9541bb71f28b 2024-11-11T07:23:12,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b2e5c5bdd6eb4c6380d8952dfe92e6e4 2024-11-11T07:23:12,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67ada2d31ff54098bb3e2f0ee6a5d66b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67ada2d31ff54098bb3e2f0ee6a5d66b 2024-11-11T07:23:12,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/5fd9500c493b44bf82c795faf68fa3b7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/5fd9500c493b44bf82c795faf68fa3b7 2024-11-11T07:23:12,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/7672f9e4d4eb4604b457b573759bed9c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/7672f9e4d4eb4604b457b573759bed9c 2024-11-11T07:23:12,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/556da1bf4931452d81dce78dd4f96b5c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/556da1bf4931452d81dce78dd4f96b5c 2024-11-11T07:23:12,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/264114cd40bf40bb9d01b2c9c258bc39 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/264114cd40bf40bb9d01b2c9c258bc39 2024-11-11T07:23:12,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1d23bced118f48598a0deb3de694585c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1d23bced118f48598a0deb3de694585c 2024-11-11T07:23:12,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b7d72312993c4d7a83064e609920db06 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/b7d72312993c4d7a83064e609920db06 2024-11-11T07:23:12,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/430b796611704d13a6e26e27024738d3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/430b796611704d13a6e26e27024738d3 2024-11-11T07:23:12,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/79840279258b4247b5e402fe212b245d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/79840279258b4247b5e402fe212b245d 2024-11-11T07:23:12,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1574f0966a0946aa8c1545e8dbcd964e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/1574f0966a0946aa8c1545e8dbcd964e 2024-11-11T07:23:12,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e93883a39e184ba59b905ab8ec7cc957 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e93883a39e184ba59b905ab8ec7cc957 2024-11-11T07:23:12,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/6e99be0190494f08b6c917776a276d4c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/6e99be0190494f08b6c917776a276d4c 2024-11-11T07:23:12,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/cd430f9a9d554a37b9fab4f53e8562f4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/cd430f9a9d554a37b9fab4f53e8562f4 2024-11-11T07:23:12,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/2493572ebedf43208b7413e1a17d4f5a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/2493572ebedf43208b7413e1a17d4f5a 2024-11-11T07:23:12,582 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=4 2024-11-11T07:23:12,583 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. 2024-11-11T07:23:12,583 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 8315bbc46c6a14c3758164d93d7c2260: Waiting for close lock at 1731309790856Running coprocessor pre-close hooks at 1731309790856Disabling compacts and flushes for region at 1731309790856Disabling writes for close at 1731309790857 (+1 ms)Obtaining lock to block concurrent updates at 1731309790857Preparing flush snapshotting stores in 8315bbc46c6a14c3758164d93d7c2260 at 1731309790857Finished memstore snapshotting TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731309790857Flushing stores of TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260. at 1731309790858 (+1 ms)Flushing 8315bbc46c6a14c3758164d93d7c2260/A: creating writer at 1731309790858Flushing mob file 8315bbc46c6a14c3758164d93d7c2260/A: appending metadata at 1731309790863 (+5 ms)Flushing mob file 8315bbc46c6a14c3758164d93d7c2260/A: closing flushed file at 1731309790863Flushing 8315bbc46c6a14c3758164d93d7c2260/A: appending metadata at 1731309791272 (+409 ms)Flushing 8315bbc46c6a14c3758164d93d7c2260/A: closing flushed file at 1731309791272Flushing 8315bbc46c6a14c3758164d93d7c2260/B: creating writer at 1731309791681 (+409 ms)Flushing 8315bbc46c6a14c3758164d93d7c2260/B: appending metadata at 1731309791683 (+2 ms)Flushing 8315bbc46c6a14c3758164d93d7c2260/B: closing flushed file at 1731309791683Flushing 8315bbc46c6a14c3758164d93d7c2260/C: creating writer at 1731309792091 (+408 ms)Flushing 8315bbc46c6a14c3758164d93d7c2260/C: appending metadata at 1731309792094 (+3 ms)Flushing 8315bbc46c6a14c3758164d93d7c2260/C: closing flushed file at 1731309792094Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3568dc0: reopening flushed file at 1731309792502 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26d957a2: reopening flushed file at 1731309792506 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32894e4b: reopening flushed file at 1731309792509 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8315bbc46c6a14c3758164d93d7c2260 in 1657ms, sequenceid=346, compaction requested=true at 1731309792514 (+5 ms)Writing region close event to WAL at 1731309792579 (+65 ms)Running coprocessor post-close hooks at 1731309792583 (+4 ms)Closed at 1731309792583 2024-11-11T07:23:12,585 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,585 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=8315bbc46c6a14c3758164d93d7c2260, regionState=CLOSED 2024-11-11T07:23:12,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:23:12,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T07:23:12,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 8315bbc46c6a14c3758164d93d7c2260, server=f334e8b0fbdf,39329,1731309728191 in 1.8830 sec 2024-11-11T07:23:12,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-11T07:23:12,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8315bbc46c6a14c3758164d93d7c2260, UNASSIGN in 1.8870 sec 2024-11-11T07:23:12,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T07:23:12,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8900 sec 2024-11-11T07:23:12,593 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309792593"}]},"ts":"1731309792593"} 2024-11-11T07:23:12,594 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:23:12,594 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:23:12,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8990 sec 2024-11-11T07:23:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T07:23:12,842 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:12,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:23:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:12,844 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:23:12,845 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:12,847 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,849 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/recovered.edits] 2024-11-11T07:23:12,852 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/9bb4b9cec5544b28b47bd44957065501 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/9bb4b9cec5544b28b47bd44957065501 2024-11-11T07:23:12,853 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a869f8f47d96406180fc361b6c2a3230 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/a869f8f47d96406180fc361b6c2a3230 2024-11-11T07:23:12,854 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/e8fe63cadd0440c09b2bf1b60255c511 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/A/e8fe63cadd0440c09b2bf1b60255c511 2024-11-11T07:23:12,856 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/08ebeba424214b068b335e5284783485 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/08ebeba424214b068b335e5284783485 2024-11-11T07:23:12,857 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/434e6119d1924a98ad78f31b2fe7292f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/434e6119d1924a98ad78f31b2fe7292f 2024-11-11T07:23:12,858 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/9171fc83c0f944819ed84c2e010f4f7c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/B/9171fc83c0f944819ed84c2e010f4f7c 2024-11-11T07:23:12,860 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/3a0f13e9d03f458f81160ff3b72691bb to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/3a0f13e9d03f458f81160ff3b72691bb 2024-11-11T07:23:12,861 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67c1239312ae4573be94a2549200980c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/67c1239312ae4573be94a2549200980c 2024-11-11T07:23:12,862 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e47745aa900544e9b7067364f01e2524 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/C/e47745aa900544e9b7067364f01e2524 2024-11-11T07:23:12,865 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/recovered.edits/349.seqid to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260/recovered.edits/349.seqid 2024-11-11T07:23:12,865 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,865 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:23:12,866 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:23:12,866 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T07:23:12,870 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111281db33b6a284d0b97e70d98882e4de0_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111281db33b6a284d0b97e70d98882e4de0_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,871 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112965223ab0a64980a6fc6fbf552b937f_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112965223ab0a64980a6fc6fbf552b937f_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,872 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113dd4bf8179944d279685a01baa8a3913_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113dd4bf8179944d279685a01baa8a3913_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,874 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111140034bc6975b40519446d525dfe794ea_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111140034bc6975b40519446d525dfe794ea_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,875 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114cd5bc2477ca4c8cb104e80a68e07a66_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114cd5bc2477ca4c8cb104e80a68e07a66_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,876 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111597f13d3d8664f479ac4b171bcb83445_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111597f13d3d8664f479ac4b171bcb83445_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,878 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111159dddae4859544748fdbca6321a32051_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111159dddae4859544748fdbca6321a32051_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,879 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d5acf7fa5f0415b9f8f2e0e568124df_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d5acf7fa5f0415b9f8f2e0e568124df_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,880 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169d566fda11b49cf85ec1abe6f42cf2d_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169d566fda11b49cf85ec1abe6f42cf2d_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,881 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d7c82f1c3864dfdbce0815675adce99_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d7c82f1c3864dfdbce0815675adce99_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,882 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119166a35848ee4d8b8575cc77930f82ca_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119166a35848ee4d8b8575cc77930f82ca_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,883 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119ec319b846114cd196fbee99800d49bd_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119ec319b846114cd196fbee99800d49bd_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,884 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac31508a3c274850a72ea9d85ee1a376_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ac31508a3c274850a72ea9d85ee1a376_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,885 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b935f867b38a4748bb5760561e139610_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b935f867b38a4748bb5760561e139610_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,886 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf758028df9b4021aa45c3650764bcaa_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf758028df9b4021aa45c3650764bcaa_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,888 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c5a4624059294b56ab5f87c372425a0a_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c5a4624059294b56ab5f87c372425a0a_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,889 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c72741432f0c40319a01ea29faa72e4e_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c72741432f0c40319a01ea29faa72e4e_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,890 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cde92f3f4f97491d8bff76df6a482381_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cde92f3f4f97491d8bff76df6a482381_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,891 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ec018406a65144e8ae1ba00c7b6ef58b_8315bbc46c6a14c3758164d93d7c2260 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ec018406a65144e8ae1ba00c7b6ef58b_8315bbc46c6a14c3758164d93d7c2260 2024-11-11T07:23:12,892 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:23:12,895 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:12,897 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:23:12,899 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:23:12,900 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:12,900 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:23:12,901 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731309792900"}]},"ts":"9223372036854775807"} 2024-11-11T07:23:12,903 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:23:12,903 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8315bbc46c6a14c3758164d93d7c2260, NAME => 'TestAcidGuarantees,,1731309758505.8315bbc46c6a14c3758164d93d7c2260.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:23:12,903 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:23:12,903 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731309792903"}]},"ts":"9223372036854775807"} 2024-11-11T07:23:12,904 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:23:12,905 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:12,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-11T07:23:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:23:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:23:12,952 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:12,961 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 326) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8084 (was 8338) 2024-11-11T07:23:12,969 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=8084 2024-11-11T07:23:12,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:23:12,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:23:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:12,973 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:23:12,973 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:12,973 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-11T07:23:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T07:23:12,973 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:23:12,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742058_1234 (size=959) 2024-11-11T07:23:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T07:23:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T07:23:13,381 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:23:13,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742059_1235 (size=53) 2024-11-11T07:23:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T07:23:13,786 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:23:13,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c49bcd3ff91d850f5acc98ee12d10ac3, disabling compactions & flushes 2024-11-11T07:23:13,787 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:13,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:13,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. after waiting 0 ms 2024-11-11T07:23:13,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:13,787 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:13,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c49bcd3ff91d850f5acc98ee12d10ac3: Waiting for close lock at 1731309793787Disabling compacts and flushes for region at 1731309793787Disabling writes for close at 1731309793787Writing region close event to WAL at 1731309793787Closed at 1731309793787 2024-11-11T07:23:13,788 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:23:13,788 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731309793788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731309793788"}]},"ts":"1731309793788"} 2024-11-11T07:23:13,789 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:23:13,790 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:23:13,790 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309793790"}]},"ts":"1731309793790"} 2024-11-11T07:23:13,791 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:23:13,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, ASSIGN}] 2024-11-11T07:23:13,793 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, ASSIGN 2024-11-11T07:23:13,793 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, ASSIGN; state=OFFLINE, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=false 2024-11-11T07:23:13,944 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c49bcd3ff91d850f5acc98ee12d10ac3, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:13,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, ASSIGN because future has completed 2024-11-11T07:23:13,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:23:14,100 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:14,100 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:23:14,101 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,101 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:23:14,101 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,101 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,102 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,103 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49bcd3ff91d850f5acc98ee12d10ac3 columnFamilyName A 2024-11-11T07:23:14,103 DEBUG [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:14,104 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] regionserver.HStore(327): Store=c49bcd3ff91d850f5acc98ee12d10ac3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:14,104 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,105 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49bcd3ff91d850f5acc98ee12d10ac3 columnFamilyName B 2024-11-11T07:23:14,105 DEBUG [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:14,105 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] regionserver.HStore(327): Store=c49bcd3ff91d850f5acc98ee12d10ac3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:14,105 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,106 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49bcd3ff91d850f5acc98ee12d10ac3 columnFamilyName C 2024-11-11T07:23:14,106 DEBUG [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:14,106 INFO [StoreOpener-c49bcd3ff91d850f5acc98ee12d10ac3-1 {}] regionserver.HStore(327): Store=c49bcd3ff91d850f5acc98ee12d10ac3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:14,107 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,107 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,107 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,108 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,108 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,109 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:23:14,109 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T07:23:14,111 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:23:14,112 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened c49bcd3ff91d850f5acc98ee12d10ac3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60185819, jitterRate=-0.10316140949726105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:23:14,112 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:14,112 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for c49bcd3ff91d850f5acc98ee12d10ac3: Running coprocessor pre-open hook at 1731309794101Writing region info on filesystem at 1731309794101Initializing all the Stores at 1731309794101Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309794102 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309794102Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309794102Cleaning up temporary data from old regions at 1731309794108 (+6 ms)Running coprocessor post-open hooks at 1731309794112 (+4 ms)Region opened successfully at 1731309794112 2024-11-11T07:23:14,113 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., pid=65, masterSystemTime=1731309794097 2024-11-11T07:23:14,115 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:14,115 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:14,115 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c49bcd3ff91d850f5acc98ee12d10ac3, regionState=OPEN, openSeqNum=2, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:14,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:23:14,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T07:23:14,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 in 171 msec 2024-11-11T07:23:14,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-11T07:23:14,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, ASSIGN in 326 msec 2024-11-11T07:23:14,120 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:23:14,121 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309794120"}]},"ts":"1731309794120"} 2024-11-11T07:23:14,122 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:23:14,122 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:23:14,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-11T07:23:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T07:23:15,122 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:15,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,124 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec7ee68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,126 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d60edd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,127 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:23:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f334e8b0fbdf,39865,1731309727465 2024-11-11T07:23:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2193c3ac 2024-11-11T07:23:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:23:15,131 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:23:15,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,132 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@485a39a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,134 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79634d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,135 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64dc14af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,136 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19897c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,138 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dd223c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,139 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345c73d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,141 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e997c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,142 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c674ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,144 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7577b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9dac7d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,146 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bddad07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,148 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f201920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,150 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1a2a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,151 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b7e4618, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,153 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c25519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,154 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa07b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143a3089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,158 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346f1c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:15,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:15,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7f931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:15,160 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221e7e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:15,161 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,161 DEBUG [Thread-1116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,162 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,162 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,162 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,162 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T07:23:15,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:23:15,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,165 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:15,166 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,166 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,167 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,168 DEBUG [Thread-1118 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,169 DEBUG [Thread-1105 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:15,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,169 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:15,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,170 DEBUG [Thread-1105 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:23:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:15,180 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:15,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:23:15,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/db6b5946003740349ed994c0cd1a63e7 is 50, key is test_row_0/A:col10/1731309795180/Put/seqid=0 2024-11-11T07:23:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309855208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309855213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309855213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309855215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309855217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742060_1236 (size=12001) 2024-11-11T07:23:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:23:15,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309855321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309855322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309855322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:23:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:15,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309855332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309855332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:23:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:15,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:15,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:15,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:23:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309855532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309855532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309855532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309855541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309855541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/db6b5946003740349ed994c0cd1a63e7 2024-11-11T07:23:15,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:23:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:15,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:15,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/b166abe8b55a41b1af699004798c61d1 is 50, key is test_row_0/B:col10/1731309795180/Put/seqid=0 2024-11-11T07:23:15,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742061_1237 (size=12001) 2024-11-11T07:23:15,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/b166abe8b55a41b1af699004798c61d1 2024-11-11T07:23:15,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/0310d27720c54cc58a26c9561dfa8447 is 50, key is test_row_0/C:col10/1731309795180/Put/seqid=0 2024-11-11T07:23:15,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742062_1238 (size=12001) 2024-11-11T07:23:15,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/0310d27720c54cc58a26c9561dfa8447 2024-11-11T07:23:15,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/db6b5946003740349ed994c0cd1a63e7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/db6b5946003740349ed994c0cd1a63e7 2024-11-11T07:23:15,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/db6b5946003740349ed994c0cd1a63e7, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T07:23:15,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/b166abe8b55a41b1af699004798c61d1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/b166abe8b55a41b1af699004798c61d1 2024-11-11T07:23:15,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/b166abe8b55a41b1af699004798c61d1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T07:23:15,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/0310d27720c54cc58a26c9561dfa8447 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0310d27720c54cc58a26c9561dfa8447 2024-11-11T07:23:15,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0310d27720c54cc58a26c9561dfa8447, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T07:23:15,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c49bcd3ff91d850f5acc98ee12d10ac3 in 546ms, sequenceid=14, compaction requested=false 2024-11-11T07:23:15,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:15,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:23:15,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:15,786 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:23:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:23:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/cbd0fdea1d094430bae205e09b41e86c is 50, key is test_row_0/A:col10/1731309795213/Put/seqid=0 2024-11-11T07:23:15,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742063_1239 (size=12001) 2024-11-11T07:23:15,798 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/cbd0fdea1d094430bae205e09b41e86c 2024-11-11T07:23:15,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fd32b9b3722448bebdc61dde3d635a77 is 50, key is test_row_0/B:col10/1731309795213/Put/seqid=0 2024-11-11T07:23:15,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742064_1240 (size=12001) 2024-11-11T07:23:15,813 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fd32b9b3722448bebdc61dde3d635a77 2024-11-11T07:23:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/8bb99cd4d4024bd6a06e99c478a5531e is 50, key is test_row_0/C:col10/1731309795213/Put/seqid=0 2024-11-11T07:23:15,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742065_1241 (size=12001) 2024-11-11T07:23:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:15,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:15,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309855852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309855853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309855854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309855855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309855856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309855962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309855962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309855971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309855971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309856172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309856172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309856181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309856181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,227 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/8bb99cd4d4024bd6a06e99c478a5531e 2024-11-11T07:23:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/cbd0fdea1d094430bae205e09b41e86c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/cbd0fdea1d094430bae205e09b41e86c 2024-11-11T07:23:16,237 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/cbd0fdea1d094430bae205e09b41e86c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T07:23:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fd32b9b3722448bebdc61dde3d635a77 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd32b9b3722448bebdc61dde3d635a77 2024-11-11T07:23:16,242 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd32b9b3722448bebdc61dde3d635a77, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T07:23:16,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/8bb99cd4d4024bd6a06e99c478a5531e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8bb99cd4d4024bd6a06e99c478a5531e 2024-11-11T07:23:16,247 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8bb99cd4d4024bd6a06e99c478a5531e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T07:23:16,248 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c49bcd3ff91d850f5acc98ee12d10ac3 in 462ms, sequenceid=37, compaction requested=false 2024-11-11T07:23:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T07:23:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T07:23:16,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T07:23:16,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0800 sec 2024-11-11T07:23:16,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.0900 sec 2024-11-11T07:23:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:23:16,303 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:16,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T07:23:16,306 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:16,306 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:16,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:23:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:16,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:16,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:23:16,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/967e0bfb9dde48c99cf342fc5b0973a4 is 50, key is test_row_0/A:col10/1731309795852/Put/seqid=0 2024-11-11T07:23:16,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742066_1242 (size=14341) 2024-11-11T07:23:16,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/967e0bfb9dde48c99cf342fc5b0973a4 2024-11-11T07:23:16,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/704217775072467caa7151e5745ccb9a is 50, key is test_row_0/B:col10/1731309795852/Put/seqid=0 2024-11-11T07:23:16,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742067_1243 (size=12001) 2024-11-11T07:23:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:23:16,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:23:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309856471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309856481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309856482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309856492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309856492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309856582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:23:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:23:16,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:23:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309856792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/704217775072467caa7151e5745ccb9a 2024-11-11T07:23:16,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/a1060117a66b4bfa983b62fadf4d37f4 is 50, key is test_row_0/C:col10/1731309795852/Put/seqid=0 2024-11-11T07:23:16,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742068_1244 (size=12001) 2024-11-11T07:23:16,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:23:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:16,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:23:16,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309856991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309856991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309857002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:17,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309857002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:17,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:23:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:17,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309857102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:17,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:23:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/a1060117a66b4bfa983b62fadf4d37f4 2024-11-11T07:23:17,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:17,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/967e0bfb9dde48c99cf342fc5b0973a4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/967e0bfb9dde48c99cf342fc5b0973a4 2024-11-11T07:23:17,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/967e0bfb9dde48c99cf342fc5b0973a4, entries=200, sequenceid=51, filesize=14.0 K 2024-11-11T07:23:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/704217775072467caa7151e5745ccb9a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/704217775072467caa7151e5745ccb9a 2024-11-11T07:23:17,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/704217775072467caa7151e5745ccb9a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T07:23:17,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/a1060117a66b4bfa983b62fadf4d37f4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/a1060117a66b4bfa983b62fadf4d37f4 2024-11-11T07:23:17,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/a1060117a66b4bfa983b62fadf4d37f4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T07:23:17,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c49bcd3ff91d850f5acc98ee12d10ac3 in 883ms, sequenceid=51, compaction requested=true 2024-11-11T07:23:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:17,247 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:17,247 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:17,249 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:17,249 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:17,249 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:17,249 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:17,249 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,249 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,249 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/b166abe8b55a41b1af699004798c61d1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd32b9b3722448bebdc61dde3d635a77, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/704217775072467caa7151e5745ccb9a] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=35.2 K 2024-11-11T07:23:17,249 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/db6b5946003740349ed994c0cd1a63e7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/cbd0fdea1d094430bae205e09b41e86c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/967e0bfb9dde48c99cf342fc5b0973a4] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=37.4 K 2024-11-11T07:23:17,250 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting db6b5946003740349ed994c0cd1a63e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731309795177 2024-11-11T07:23:17,250 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b166abe8b55a41b1af699004798c61d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731309795177 2024-11-11T07:23:17,250 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd0fdea1d094430bae205e09b41e86c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731309795210 2024-11-11T07:23:17,250 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting fd32b9b3722448bebdc61dde3d635a77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731309795210 2024-11-11T07:23:17,250 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 704217775072467caa7151e5745ccb9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731309795852 2024-11-11T07:23:17,251 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 967e0bfb9dde48c99cf342fc5b0973a4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731309795848 2024-11-11T07:23:17,259 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:17,260 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/919a4c780fb446fb884f0f34d595e595 is 50, key is test_row_0/B:col10/1731309795852/Put/seqid=0 2024-11-11T07:23:17,263 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:17,263 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2a5efd5bdaee480b80af2edc679dea16 is 50, key is test_row_0/A:col10/1731309795852/Put/seqid=0 2024-11-11T07:23:17,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742069_1245 (size=12104) 2024-11-11T07:23:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742070_1246 (size=12104) 2024-11-11T07:23:17,301 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2a5efd5bdaee480b80af2edc679dea16 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2a5efd5bdaee480b80af2edc679dea16 2024-11-11T07:23:17,309 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into 2a5efd5bdaee480b80af2edc679dea16(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:17,309 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:17,309 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=13, startTime=1731309797247; duration=0sec 2024-11-11T07:23:17,310 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:17,310 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:17,310 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:17,311 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:17,311 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:17,311 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,312 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0310d27720c54cc58a26c9561dfa8447, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8bb99cd4d4024bd6a06e99c478a5531e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/a1060117a66b4bfa983b62fadf4d37f4] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=35.2 K 2024-11-11T07:23:17,312 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0310d27720c54cc58a26c9561dfa8447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731309795177 2024-11-11T07:23:17,313 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb99cd4d4024bd6a06e99c478a5531e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731309795210 2024-11-11T07:23:17,313 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1060117a66b4bfa983b62fadf4d37f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731309795852 2024-11-11T07:23:17,323 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:17,324 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/da96988635d148ae8f2387b07c08f9c3 is 50, key is test_row_0/C:col10/1731309795852/Put/seqid=0 2024-11-11T07:23:17,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742071_1247 (size=12104) 2024-11-11T07:23:17,350 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/da96988635d148ae8f2387b07c08f9c3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/da96988635d148ae8f2387b07c08f9c3 2024-11-11T07:23:17,357 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into da96988635d148ae8f2387b07c08f9c3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:17,357 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:17,357 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=13, startTime=1731309797247; duration=0sec 2024-11-11T07:23:17,357 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:17,357 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:17,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:23:17,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,380 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T07:23:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/415e1391edeb4d55b038b2ac558ff97a is 50, key is test_row_0/A:col10/1731309796465/Put/seqid=0 2024-11-11T07:23:17,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742072_1248 (size=12001) 2024-11-11T07:23:17,390 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/415e1391edeb4d55b038b2ac558ff97a 2024-11-11T07:23:17,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/2fcc252179194fea8bf51fbbb7f1525b is 50, key is test_row_0/B:col10/1731309796465/Put/seqid=0 2024-11-11T07:23:17,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742073_1249 (size=12001) 2024-11-11T07:23:17,404 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/2fcc252179194fea8bf51fbbb7f1525b 2024-11-11T07:23:17,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2a3122ce187a4382bf09e0dd7b0e69a6 is 50, key is test_row_0/C:col10/1731309796465/Put/seqid=0 2024-11-11T07:23:17,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742074_1250 (size=12001) 2024-11-11T07:23:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:23:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:17,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:17,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309857665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:17,681 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/919a4c780fb446fb884f0f34d595e595 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/919a4c780fb446fb884f0f34d595e595 2024-11-11T07:23:17,686 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into 919a4c780fb446fb884f0f34d595e595(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:17,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:17,686 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=13, startTime=1731309797247; duration=0sec 2024-11-11T07:23:17,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:17,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:17,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309857771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:17,818 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2a3122ce187a4382bf09e0dd7b0e69a6 2024-11-11T07:23:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/415e1391edeb4d55b038b2ac558ff97a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/415e1391edeb4d55b038b2ac558ff97a 2024-11-11T07:23:17,829 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/415e1391edeb4d55b038b2ac558ff97a, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T07:23:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/2fcc252179194fea8bf51fbbb7f1525b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/2fcc252179194fea8bf51fbbb7f1525b 2024-11-11T07:23:17,836 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/2fcc252179194fea8bf51fbbb7f1525b, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T07:23:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2a3122ce187a4382bf09e0dd7b0e69a6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2a3122ce187a4382bf09e0dd7b0e69a6 2024-11-11T07:23:17,842 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2a3122ce187a4382bf09e0dd7b0e69a6, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T07:23:17,843 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for c49bcd3ff91d850f5acc98ee12d10ac3 in 463ms, sequenceid=75, compaction requested=false 2024-11-11T07:23:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:17,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T07:23:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T07:23:17,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T07:23:17,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5380 sec 2024-11-11T07:23:17,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.5440 sec 2024-11-11T07:23:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:17,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:17,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T07:23:17,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/4821a59d46944279aa6c98005c9669c6 is 50, key is test_row_0/A:col10/1731309797983/Put/seqid=0 2024-11-11T07:23:17,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742075_1251 (size=14341) 2024-11-11T07:23:17,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/4821a59d46944279aa6c98005c9669c6 2024-11-11T07:23:18,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/7661686c138743148896988a4f48f7b2 is 50, key is test_row_0/B:col10/1731309797983/Put/seqid=0 2024-11-11T07:23:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742076_1252 (size=12001) 2024-11-11T07:23:18,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/7661686c138743148896988a4f48f7b2 2024-11-11T07:23:18,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/d5caed4041f44f1dad43b1825da54a68 is 50, key is test_row_0/C:col10/1731309797983/Put/seqid=0 2024-11-11T07:23:18,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742077_1253 (size=12001) 2024-11-11T07:23:18,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309858028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309858028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309858029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309858030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309858033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,106 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:23:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309858142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309858142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309858142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309858142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309858142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309858351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309858351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309858351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309858351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309858351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/d5caed4041f44f1dad43b1825da54a68 2024-11-11T07:23:18,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/4821a59d46944279aa6c98005c9669c6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4821a59d46944279aa6c98005c9669c6 2024-11-11T07:23:18,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4821a59d46944279aa6c98005c9669c6, entries=200, sequenceid=91, filesize=14.0 K 2024-11-11T07:23:18,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/7661686c138743148896988a4f48f7b2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/7661686c138743148896988a4f48f7b2 2024-11-11T07:23:18,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/7661686c138743148896988a4f48f7b2, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T07:23:18,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/d5caed4041f44f1dad43b1825da54a68 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/d5caed4041f44f1dad43b1825da54a68 2024-11-11T07:23:18,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/d5caed4041f44f1dad43b1825da54a68, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T07:23:18,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c49bcd3ff91d850f5acc98ee12d10ac3 in 464ms, sequenceid=91, compaction requested=true 2024-11-11T07:23:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:18,448 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:18,448 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:18,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:18,450 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:18,450 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:18,451 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:18,451 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:18,451 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,451 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,451 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/919a4c780fb446fb884f0f34d595e595, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/2fcc252179194fea8bf51fbbb7f1525b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/7661686c138743148896988a4f48f7b2] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=35.3 K 2024-11-11T07:23:18,451 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2a5efd5bdaee480b80af2edc679dea16, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/415e1391edeb4d55b038b2ac558ff97a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4821a59d46944279aa6c98005c9669c6] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=37.5 K 2024-11-11T07:23:18,451 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a5efd5bdaee480b80af2edc679dea16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731309795852 2024-11-11T07:23:18,451 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 919a4c780fb446fb884f0f34d595e595, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731309795852 2024-11-11T07:23:18,451 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 415e1391edeb4d55b038b2ac558ff97a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731309796459 2024-11-11T07:23:18,452 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fcc252179194fea8bf51fbbb7f1525b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731309796459 2024-11-11T07:23:18,452 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4821a59d46944279aa6c98005c9669c6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731309797646 2024-11-11T07:23:18,452 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7661686c138743148896988a4f48f7b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731309797657 2024-11-11T07:23:18,461 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:18,461 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:23:18,462 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/df054fb81fda4c02b62df550375b41b2 is 50, key is test_row_0/A:col10/1731309797983/Put/seqid=0 2024-11-11T07:23:18,462 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:18,462 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fc151d0b476d4b02a1ecb2c8c2f8e1c3 is 50, key is test_row_0/B:col10/1731309797983/Put/seqid=0 2024-11-11T07:23:18,462 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T07:23:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T07:23:18,465 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:18,466 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:18,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:18,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742078_1254 (size=12207) 2024-11-11T07:23:18,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742079_1255 (size=12207) 2024-11-11T07:23:18,486 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fc151d0b476d4b02a1ecb2c8c2f8e1c3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fc151d0b476d4b02a1ecb2c8c2f8e1c3 2024-11-11T07:23:18,493 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/df054fb81fda4c02b62df550375b41b2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/df054fb81fda4c02b62df550375b41b2 2024-11-11T07:23:18,494 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into fc151d0b476d4b02a1ecb2c8c2f8e1c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:18,494 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:18,494 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=13, startTime=1731309798448; duration=0sec 2024-11-11T07:23:18,494 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:18,494 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:18,494 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:18,496 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:18,496 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:18,496 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,496 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/da96988635d148ae8f2387b07c08f9c3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2a3122ce187a4382bf09e0dd7b0e69a6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/d5caed4041f44f1dad43b1825da54a68] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=35.3 K 2024-11-11T07:23:18,497 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting da96988635d148ae8f2387b07c08f9c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731309795852 2024-11-11T07:23:18,497 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3122ce187a4382bf09e0dd7b0e69a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731309796459 2024-11-11T07:23:18,498 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting d5caed4041f44f1dad43b1825da54a68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731309797657 2024-11-11T07:23:18,503 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into df054fb81fda4c02b62df550375b41b2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:18,503 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:18,503 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=13, startTime=1731309798448; duration=0sec 2024-11-11T07:23:18,504 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:18,504 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:18,507 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:18,507 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/4c20fa47525b4ab7b6faf7ff4da15d65 is 50, key is test_row_0/C:col10/1731309797983/Put/seqid=0 2024-11-11T07:23:18,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742080_1256 (size=12207) 2024-11-11T07:23:18,525 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/4c20fa47525b4ab7b6faf7ff4da15d65 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c20fa47525b4ab7b6faf7ff4da15d65 2024-11-11T07:23:18,533 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into 4c20fa47525b4ab7b6faf7ff4da15d65(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:18,533 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:18,533 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=13, startTime=1731309798448; duration=0sec 2024-11-11T07:23:18,533 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:18,533 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T07:23:18,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T07:23:18,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,619 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T07:23:18,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/5a8cc2a0305349579725f69e5b2c7503 is 50, key is test_row_0/A:col10/1731309798027/Put/seqid=0 2024-11-11T07:23:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742081_1257 (size=12001) 2024-11-11T07:23:18,634 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/5a8cc2a0305349579725f69e5b2c7503 2024-11-11T07:23:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/3b37ea73c86a470892a78444267bca18 is 50, key is test_row_0/B:col10/1731309798027/Put/seqid=0 2024-11-11T07:23:18,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742082_1258 (size=12001) 2024-11-11T07:23:18,651 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/3b37ea73c86a470892a78444267bca18 2024-11-11T07:23:18,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/ee3b6469ade847b0a323bfd786dc7c78 is 50, key is test_row_0/C:col10/1731309798027/Put/seqid=0 2024-11-11T07:23:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:18,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309858672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309858675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309858676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309858678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309858679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742083_1259 (size=12001) 2024-11-11T07:23:18,691 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/ee3b6469ade847b0a323bfd786dc7c78 2024-11-11T07:23:18,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/5a8cc2a0305349579725f69e5b2c7503 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/5a8cc2a0305349579725f69e5b2c7503 2024-11-11T07:23:18,702 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/5a8cc2a0305349579725f69e5b2c7503, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T07:23:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/3b37ea73c86a470892a78444267bca18 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3b37ea73c86a470892a78444267bca18 2024-11-11T07:23:18,707 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3b37ea73c86a470892a78444267bca18, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T07:23:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/ee3b6469ade847b0a323bfd786dc7c78 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ee3b6469ade847b0a323bfd786dc7c78 2024-11-11T07:23:18,713 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ee3b6469ade847b0a323bfd786dc7c78, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T07:23:18,714 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for c49bcd3ff91d850f5acc98ee12d10ac3 in 95ms, sequenceid=115, compaction requested=false 2024-11-11T07:23:18,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:18,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T07:23:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T07:23:18,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T07:23:18,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 249 msec 2024-11-11T07:23:18,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 256 msec 2024-11-11T07:23:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T07:23:18,781 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:18,782 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T07:23:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:23:18,784 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:18,784 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:18,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:18,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T07:23:18,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/12d61628b575416ca18ba04aa0e1beda is 50, key is test_row_0/A:col10/1731309798676/Put/seqid=0 2024-11-11T07:23:18,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742084_1260 (size=12101) 2024-11-11T07:23:18,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/12d61628b575416ca18ba04aa0e1beda 2024-11-11T07:23:18,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/748fa1eefe4f458fba3abf9d7d21d0ff is 50, key is test_row_0/B:col10/1731309798676/Put/seqid=0 2024-11-11T07:23:18,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742085_1261 (size=12101) 2024-11-11T07:23:18,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/748fa1eefe4f458fba3abf9d7d21d0ff 2024-11-11T07:23:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309858810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309858811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309858814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309858817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309858817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/4c34832285d5424cbc42aad8144d0b67 is 50, key is test_row_0/C:col10/1731309798676/Put/seqid=0 2024-11-11T07:23:18,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742086_1262 (size=12101) 2024-11-11T07:23:18,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/4c34832285d5424cbc42aad8144d0b67 2024-11-11T07:23:18,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/12d61628b575416ca18ba04aa0e1beda as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12d61628b575416ca18ba04aa0e1beda 2024-11-11T07:23:18,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12d61628b575416ca18ba04aa0e1beda, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T07:23:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/748fa1eefe4f458fba3abf9d7d21d0ff as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/748fa1eefe4f458fba3abf9d7d21d0ff 2024-11-11T07:23:18,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/748fa1eefe4f458fba3abf9d7d21d0ff, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T07:23:18,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/4c34832285d5424cbc42aad8144d0b67 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c34832285d5424cbc42aad8144d0b67 2024-11-11T07:23:18,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c34832285d5424cbc42aad8144d0b67, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T07:23:18,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for c49bcd3ff91d850f5acc98ee12d10ac3 in 92ms, sequenceid=131, compaction requested=true 2024-11-11T07:23:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:18,877 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:18,877 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:18,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:18,878 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:18,878 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:18,878 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,878 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/df054fb81fda4c02b62df550375b41b2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/5a8cc2a0305349579725f69e5b2c7503, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12d61628b575416ca18ba04aa0e1beda] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=35.5 K 2024-11-11T07:23:18,878 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:18,879 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:18,879 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,879 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fc151d0b476d4b02a1ecb2c8c2f8e1c3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3b37ea73c86a470892a78444267bca18, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/748fa1eefe4f458fba3abf9d7d21d0ff] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=35.5 K 2024-11-11T07:23:18,879 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting fc151d0b476d4b02a1ecb2c8c2f8e1c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731309797657 2024-11-11T07:23:18,879 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting df054fb81fda4c02b62df550375b41b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731309797657 2024-11-11T07:23:18,880 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b37ea73c86a470892a78444267bca18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731309798026 2024-11-11T07:23:18,880 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8cc2a0305349579725f69e5b2c7503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731309798026 2024-11-11T07:23:18,880 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 748fa1eefe4f458fba3abf9d7d21d0ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309798668 2024-11-11T07:23:18,883 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d61628b575416ca18ba04aa0e1beda, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309798668 2024-11-11T07:23:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:23:18,894 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#221 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:18,894 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:18,894 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/25fc5eac0d194cddb721b778a6d01e54 is 50, key is test_row_0/B:col10/1731309798676/Put/seqid=0 2024-11-11T07:23:18,895 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/e072f45908104f65b1290a5293a12954 is 50, key is test_row_0/A:col10/1731309798676/Put/seqid=0 2024-11-11T07:23:18,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742087_1263 (size=12409) 2024-11-11T07:23:18,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742088_1264 (size=12409) 2024-11-11T07:23:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:18,925 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:18,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T07:23:18,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2ee7edccb31045c0892061265376911c is 50, key is test_row_0/A:col10/1731309798925/Put/seqid=0 2024-11-11T07:23:18,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:18,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:18,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:18,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309858940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309858945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309858946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309858947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309858948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:18,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742089_1265 (size=14541) 2024-11-11T07:23:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309859052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309859062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309859062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309859062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309859062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:19,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:19,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:23:19,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309859261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309859271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309859271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309859272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309859272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,311 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/25fc5eac0d194cddb721b778a6d01e54 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/25fc5eac0d194cddb721b778a6d01e54 2024-11-11T07:23:19,317 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/e072f45908104f65b1290a5293a12954 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e072f45908104f65b1290a5293a12954 2024-11-11T07:23:19,318 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into 25fc5eac0d194cddb721b778a6d01e54(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:19,318 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:19,318 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=13, startTime=1731309798877; duration=0sec 2024-11-11T07:23:19,318 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:19,318 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:19,318 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:19,319 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:19,320 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:19,320 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,320 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c20fa47525b4ab7b6faf7ff4da15d65, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ee3b6469ade847b0a323bfd786dc7c78, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c34832285d5424cbc42aad8144d0b67] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=35.5 K 2024-11-11T07:23:19,321 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c20fa47525b4ab7b6faf7ff4da15d65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731309797657 2024-11-11T07:23:19,322 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3b6469ade847b0a323bfd786dc7c78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731309798026 2024-11-11T07:23:19,322 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c34832285d5424cbc42aad8144d0b67, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309798668 2024-11-11T07:23:19,322 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into e072f45908104f65b1290a5293a12954(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:19,323 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:19,323 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=13, startTime=1731309798877; duration=0sec 2024-11-11T07:23:19,323 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:19,323 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:19,329 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:19,329 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/bb4abb4f730943368434331a8201799b is 50, key is test_row_0/C:col10/1731309798676/Put/seqid=0 2024-11-11T07:23:19,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742090_1266 (size=12409) 2024-11-11T07:23:19,338 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/bb4abb4f730943368434331a8201799b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/bb4abb4f730943368434331a8201799b 2024-11-11T07:23:19,343 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into bb4abb4f730943368434331a8201799b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:19,343 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:19,343 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=13, startTime=1731309798877; duration=0sec 2024-11-11T07:23:19,343 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:19,343 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:19,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2ee7edccb31045c0892061265376911c 2024-11-11T07:23:19,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/81ffc13ab6c74ef388df14482f44e405 is 50, key is test_row_0/B:col10/1731309798925/Put/seqid=0 2024-11-11T07:23:19,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742091_1267 (size=12151) 2024-11-11T07:23:19,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:23:19,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:19,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:19,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309859571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309859581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309859581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309859581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309859582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:19,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/81ffc13ab6c74ef388df14482f44e405 2024-11-11T07:23:19,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/78c7d85c1fdb4cb7a257abd13981c3ed is 50, key is test_row_0/C:col10/1731309798925/Put/seqid=0 2024-11-11T07:23:19,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742092_1268 (size=12151) 2024-11-11T07:23:19,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:19,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:23:20,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:20,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:20,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309860081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309860092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309860092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309860092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309860092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:20,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:20,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:20,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/78c7d85c1fdb4cb7a257abd13981c3ed 2024-11-11T07:23:20,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2ee7edccb31045c0892061265376911c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2ee7edccb31045c0892061265376911c 2024-11-11T07:23:20,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2ee7edccb31045c0892061265376911c, entries=200, sequenceid=154, filesize=14.2 K 2024-11-11T07:23:20,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/81ffc13ab6c74ef388df14482f44e405 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/81ffc13ab6c74ef388df14482f44e405 2024-11-11T07:23:20,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/81ffc13ab6c74ef388df14482f44e405, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T07:23:20,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/78c7d85c1fdb4cb7a257abd13981c3ed as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/78c7d85c1fdb4cb7a257abd13981c3ed 2024-11-11T07:23:20,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/78c7d85c1fdb4cb7a257abd13981c3ed, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T07:23:20,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1277ms, sequenceid=154, compaction requested=false 2024-11-11T07:23:20,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:20,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:23:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:20,317 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:23:20,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/49d430eecdd848068f79f53dd2e87885 is 50, key is test_row_0/A:col10/1731309798937/Put/seqid=0 2024-11-11T07:23:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742093_1269 (size=12151) 2024-11-11T07:23:20,735 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/49d430eecdd848068f79f53dd2e87885 2024-11-11T07:23:20,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fd0fd015f27c46a7b7ae453da7739df9 is 50, key is test_row_0/B:col10/1731309798937/Put/seqid=0 2024-11-11T07:23:20,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742094_1270 (size=12151) 2024-11-11T07:23:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:23:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:21,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:21,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309861124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309861125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309861126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309861131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309861132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,149 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fd0fd015f27c46a7b7ae453da7739df9 2024-11-11T07:23:21,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/0dbff6c1562245e685f98086004a8b78 is 50, key is test_row_0/C:col10/1731309798937/Put/seqid=0 2024-11-11T07:23:21,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742095_1271 (size=12151) 2024-11-11T07:23:21,162 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/0dbff6c1562245e685f98086004a8b78 2024-11-11T07:23:21,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/49d430eecdd848068f79f53dd2e87885 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/49d430eecdd848068f79f53dd2e87885 2024-11-11T07:23:21,171 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/49d430eecdd848068f79f53dd2e87885, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:23:21,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/fd0fd015f27c46a7b7ae453da7739df9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd0fd015f27c46a7b7ae453da7739df9 2024-11-11T07:23:21,176 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd0fd015f27c46a7b7ae453da7739df9, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:23:21,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/0dbff6c1562245e685f98086004a8b78 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0dbff6c1562245e685f98086004a8b78 2024-11-11T07:23:21,180 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0dbff6c1562245e685f98086004a8b78, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:23:21,181 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for c49bcd3ff91d850f5acc98ee12d10ac3 in 865ms, sequenceid=171, compaction requested=true 2024-11-11T07:23:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T07:23:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T07:23:21,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T07:23:21,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3990 sec 2024-11-11T07:23:21,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.4030 sec 2024-11-11T07:23:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:21,244 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:21,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:23:21,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/bcb1fad27dfe4dc9aa1799fc91a85e60 is 50, key is test_row_0/A:col10/1731309801245/Put/seqid=0 2024-11-11T07:23:21,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742096_1272 (size=12151) 2024-11-11T07:23:21,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309861252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309861253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309861254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309861255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309861256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309861372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309861373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309861373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309861373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309861373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309861581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309861582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309861582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309861582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309861582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/bcb1fad27dfe4dc9aa1799fc91a85e60 2024-11-11T07:23:21,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/8b5230ef61344c52bfe51a1b4d7b3d36 is 50, key is test_row_0/B:col10/1731309801245/Put/seqid=0 2024-11-11T07:23:21,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742097_1273 (size=12151) 2024-11-11T07:23:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309861891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309861891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309861891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309861892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309861892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/8b5230ef61344c52bfe51a1b4d7b3d36 2024-11-11T07:23:22,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/011eb09688c2433f943f854366d6a40c is 50, key is test_row_0/C:col10/1731309801245/Put/seqid=0 2024-11-11T07:23:22,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742098_1274 (size=12151) 2024-11-11T07:23:22,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/011eb09688c2433f943f854366d6a40c 2024-11-11T07:23:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/bcb1fad27dfe4dc9aa1799fc91a85e60 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bcb1fad27dfe4dc9aa1799fc91a85e60 2024-11-11T07:23:22,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bcb1fad27dfe4dc9aa1799fc91a85e60, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T07:23:22,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/8b5230ef61344c52bfe51a1b4d7b3d36 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8b5230ef61344c52bfe51a1b4d7b3d36 2024-11-11T07:23:22,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8b5230ef61344c52bfe51a1b4d7b3d36, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T07:23:22,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/011eb09688c2433f943f854366d6a40c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/011eb09688c2433f943f854366d6a40c 2024-11-11T07:23:22,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/011eb09688c2433f943f854366d6a40c, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T07:23:22,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c49bcd3ff91d850f5acc98ee12d10ac3 in 855ms, sequenceid=196, compaction requested=true 2024-11-11T07:23:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:22,099 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:22,099 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:22,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:22,100 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51252 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:22,100 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:22,100 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:22,100 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:22,100 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:22,100 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:22,101 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/25fc5eac0d194cddb721b778a6d01e54, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/81ffc13ab6c74ef388df14482f44e405, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd0fd015f27c46a7b7ae453da7739df9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8b5230ef61344c52bfe51a1b4d7b3d36] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=47.7 K 2024-11-11T07:23:22,101 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e072f45908104f65b1290a5293a12954, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2ee7edccb31045c0892061265376911c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/49d430eecdd848068f79f53dd2e87885, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bcb1fad27dfe4dc9aa1799fc91a85e60] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=50.1 K 2024-11-11T07:23:22,101 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting e072f45908104f65b1290a5293a12954, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309798668 2024-11-11T07:23:22,101 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 25fc5eac0d194cddb721b778a6d01e54, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309798668 2024-11-11T07:23:22,102 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ffc13ab6c74ef388df14482f44e405, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731309798807 2024-11-11T07:23:22,102 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ee7edccb31045c0892061265376911c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731309798807 2024-11-11T07:23:22,102 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0fd015f27c46a7b7ae453da7739df9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731309798937 2024-11-11T07:23:22,102 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49d430eecdd848068f79f53dd2e87885, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731309798937 2024-11-11T07:23:22,102 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b5230ef61344c52bfe51a1b4d7b3d36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309801122 2024-11-11T07:23:22,102 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb1fad27dfe4dc9aa1799fc91a85e60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309801122 2024-11-11T07:23:22,113 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:22,113 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:22,114 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/82bdc00f773543659075ef42fdd12862 is 50, key is test_row_0/B:col10/1731309801245/Put/seqid=0 2024-11-11T07:23:22,114 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/d6e4d2c21e2c429ea5eb87edb2eb3842 is 50, key is test_row_0/A:col10/1731309801245/Put/seqid=0 2024-11-11T07:23:22,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742099_1275 (size=12595) 2024-11-11T07:23:22,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742100_1276 (size=12595) 2024-11-11T07:23:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:22,404 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:22,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:23:22,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/151751d3a04b453ab9cbd26afc09d0ce is 50, key is test_row_0/A:col10/1731309801254/Put/seqid=0 2024-11-11T07:23:22,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742101_1277 (size=12151) 2024-11-11T07:23:22,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/151751d3a04b453ab9cbd26afc09d0ce 2024-11-11T07:23:22,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/856b23cb6fa145f6981d282f9670afe2 is 50, key is test_row_0/B:col10/1731309801254/Put/seqid=0 2024-11-11T07:23:22,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742102_1278 (size=12151) 2024-11-11T07:23:22,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309862429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309862431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309862432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309862432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309862433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,534 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/82bdc00f773543659075ef42fdd12862 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/82bdc00f773543659075ef42fdd12862 2024-11-11T07:23:22,537 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/d6e4d2c21e2c429ea5eb87edb2eb3842 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d6e4d2c21e2c429ea5eb87edb2eb3842 2024-11-11T07:23:22,539 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into 82bdc00f773543659075ef42fdd12862(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:22,539 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:22,539 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=12, startTime=1731309802099; duration=0sec 2024-11-11T07:23:22,539 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:22,539 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:22,539 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:22,540 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:22,540 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:22,541 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:22,541 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/bb4abb4f730943368434331a8201799b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/78c7d85c1fdb4cb7a257abd13981c3ed, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0dbff6c1562245e685f98086004a8b78, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/011eb09688c2433f943f854366d6a40c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=47.7 K 2024-11-11T07:23:22,541 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting bb4abb4f730943368434331a8201799b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309798668 2024-11-11T07:23:22,542 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c7d85c1fdb4cb7a257abd13981c3ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731309798807 2024-11-11T07:23:22,542 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dbff6c1562245e685f98086004a8b78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731309798937 2024-11-11T07:23:22,543 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 011eb09688c2433f943f854366d6a40c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309801122 2024-11-11T07:23:22,543 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into d6e4d2c21e2c429ea5eb87edb2eb3842(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:22,543 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:22,543 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=12, startTime=1731309802099; duration=0sec 2024-11-11T07:23:22,543 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:22,543 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309862541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309862542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309862543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309862544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,551 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:22,552 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/04e0fc5478bb4af6b4b49bae4c090361 is 50, key is test_row_0/C:col10/1731309801245/Put/seqid=0 2024-11-11T07:23:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309862551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742103_1279 (size=12595) 2024-11-11T07:23:22,564 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/04e0fc5478bb4af6b4b49bae4c090361 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/04e0fc5478bb4af6b4b49bae4c090361 2024-11-11T07:23:22,569 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into 04e0fc5478bb4af6b4b49bae4c090361(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:22,569 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:22,569 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=12, startTime=1731309802099; duration=0sec 2024-11-11T07:23:22,569 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:22,569 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:22,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309862752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309862752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309862752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309862752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309862761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:22,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/856b23cb6fa145f6981d282f9670afe2 2024-11-11T07:23:22,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/135821b0cd544a6096536e0a7a0f8ca2 is 50, key is test_row_0/C:col10/1731309801254/Put/seqid=0 2024-11-11T07:23:22,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742104_1280 (size=12151) 2024-11-11T07:23:22,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/135821b0cd544a6096536e0a7a0f8ca2 2024-11-11T07:23:22,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/151751d3a04b453ab9cbd26afc09d0ce as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/151751d3a04b453ab9cbd26afc09d0ce 2024-11-11T07:23:22,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/151751d3a04b453ab9cbd26afc09d0ce, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:23:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/856b23cb6fa145f6981d282f9670afe2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/856b23cb6fa145f6981d282f9670afe2 2024-11-11T07:23:22,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/856b23cb6fa145f6981d282f9670afe2, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:23:22,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/135821b0cd544a6096536e0a7a0f8ca2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/135821b0cd544a6096536e0a7a0f8ca2 2024-11-11T07:23:22,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/135821b0cd544a6096536e0a7a0f8ca2, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:23:22,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c49bcd3ff91d850f5acc98ee12d10ac3 in 459ms, sequenceid=209, compaction requested=false 2024-11-11T07:23:22,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:23:22,971 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:22,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T07:23:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:23:22,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:22,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:22,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:23,065 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:23,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:23:23,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/36d9a18a843945e6ac7dbc607b941d8d is 50, key is test_row_0/A:col10/1731309803065/Put/seqid=0 2024-11-11T07:23:23,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309863071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309863071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309863072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309863073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309863075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:23:23,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742105_1281 (size=16931) 2024-11-11T07:23:23,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309863181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309863182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309863182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309863182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:23,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:23:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309863392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309863392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309863392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309863392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:23,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:23,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/36d9a18a843945e6ac7dbc607b941d8d 2024-11-11T07:23:23,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/531e5afd94c04fd99b3b6d0eedd1c53a is 50, key is test_row_0/B:col10/1731309803065/Put/seqid=0 2024-11-11T07:23:23,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742106_1282 (size=12151) 2024-11-11T07:23:23,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309863581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:23:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309863701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309863702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309863702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309863702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:23,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:23,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:23,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:23,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:23,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/531e5afd94c04fd99b3b6d0eedd1c53a 2024-11-11T07:23:23,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/76f24866f576465d97dfb3622b5ff77a is 50, key is test_row_0/C:col10/1731309803065/Put/seqid=0 2024-11-11T07:23:23,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742107_1283 (size=12151) 2024-11-11T07:23:24,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:24,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:24,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:24,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:23:24,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:24,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309864212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:24,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309864212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309864212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:24,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309864212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:24,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:24,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/76f24866f576465d97dfb3622b5ff77a 2024-11-11T07:23:24,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/36d9a18a843945e6ac7dbc607b941d8d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/36d9a18a843945e6ac7dbc607b941d8d 2024-11-11T07:23:24,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/36d9a18a843945e6ac7dbc607b941d8d, entries=250, sequenceid=239, filesize=16.5 K 2024-11-11T07:23:24,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/531e5afd94c04fd99b3b6d0eedd1c53a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/531e5afd94c04fd99b3b6d0eedd1c53a 2024-11-11T07:23:24,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/531e5afd94c04fd99b3b6d0eedd1c53a, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T07:23:24,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/76f24866f576465d97dfb3622b5ff77a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/76f24866f576465d97dfb3622b5ff77a 2024-11-11T07:23:24,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/76f24866f576465d97dfb3622b5ff77a, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T07:23:24,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1261ms, sequenceid=239, compaction requested=true 2024-11-11T07:23:24,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:24,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:24,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:24,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:24,327 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:24,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:24,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:24,327 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:24,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:24,328 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:24,328 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:24,328 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:24,328 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:24,328 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,328 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,328 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/82bdc00f773543659075ef42fdd12862, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/856b23cb6fa145f6981d282f9670afe2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/531e5afd94c04fd99b3b6d0eedd1c53a] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=36.0 K 2024-11-11T07:23:24,328 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d6e4d2c21e2c429ea5eb87edb2eb3842, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/151751d3a04b453ab9cbd26afc09d0ce, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/36d9a18a843945e6ac7dbc607b941d8d] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=40.7 K 2024-11-11T07:23:24,328 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 82bdc00f773543659075ef42fdd12862, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309801122 2024-11-11T07:23:24,328 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6e4d2c21e2c429ea5eb87edb2eb3842, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309801122 2024-11-11T07:23:24,329 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 856b23cb6fa145f6981d282f9670afe2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309801254 2024-11-11T07:23:24,329 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 151751d3a04b453ab9cbd26afc09d0ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309801254 2024-11-11T07:23:24,329 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 531e5afd94c04fd99b3b6d0eedd1c53a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731309802431 2024-11-11T07:23:24,330 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d9a18a843945e6ac7dbc607b941d8d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731309802429 2024-11-11T07:23:24,337 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:24,338 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/aba80b83afa44b7dbd30cc36480492c3 is 50, key is test_row_0/A:col10/1731309803065/Put/seqid=0 2024-11-11T07:23:24,340 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:24,341 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/9a7628f3f4f240a88abd2c1df63419eb is 50, key is test_row_0/B:col10/1731309803065/Put/seqid=0 2024-11-11T07:23:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742108_1284 (size=12697) 2024-11-11T07:23:24,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:23:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T07:23:24,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/7591de80bbb147848a626ebd33afc65b is 50, key is test_row_0/A:col10/1731309803074/Put/seqid=0 2024-11-11T07:23:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742109_1285 (size=12697) 2024-11-11T07:23:24,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742110_1286 (size=12151) 2024-11-11T07:23:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:24,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:24,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309864705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:24,755 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/aba80b83afa44b7dbd30cc36480492c3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/aba80b83afa44b7dbd30cc36480492c3 2024-11-11T07:23:24,760 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into aba80b83afa44b7dbd30cc36480492c3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:24,760 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:24,760 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=13, startTime=1731309804326; duration=0sec 2024-11-11T07:23:24,760 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:24,760 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:24,760 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:24,761 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:24,761 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:24,761 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:24,761 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/04e0fc5478bb4af6b4b49bae4c090361, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/135821b0cd544a6096536e0a7a0f8ca2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/76f24866f576465d97dfb3622b5ff77a] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=36.0 K 2024-11-11T07:23:24,761 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04e0fc5478bb4af6b4b49bae4c090361, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309801122 2024-11-11T07:23:24,762 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135821b0cd544a6096536e0a7a0f8ca2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309801254 2024-11-11T07:23:24,762 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76f24866f576465d97dfb3622b5ff77a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731309802431 2024-11-11T07:23:24,770 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/7591de80bbb147848a626ebd33afc65b 2024-11-11T07:23:24,775 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/9a7628f3f4f240a88abd2c1df63419eb as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/9a7628f3f4f240a88abd2c1df63419eb 2024-11-11T07:23:24,775 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:24,775 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/8dbe8f01f71640d3a7f7bc892eabbad2 is 50, key is test_row_0/C:col10/1731309803065/Put/seqid=0 2024-11-11T07:23:24,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/d418fef56007427a84da88b378c9ab69 is 50, key is test_row_0/B:col10/1731309803074/Put/seqid=0 2024-11-11T07:23:24,779 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into 9a7628f3f4f240a88abd2c1df63419eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:24,779 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:24,779 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=13, startTime=1731309804327; duration=0sec 2024-11-11T07:23:24,779 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:24,780 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:24,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742111_1287 (size=12697) 2024-11-11T07:23:24,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742112_1288 (size=12151) 2024-11-11T07:23:24,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309864812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:24,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309865021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:23:25,187 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/8dbe8f01f71640d3a7f7bc892eabbad2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8dbe8f01f71640d3a7f7bc892eabbad2 2024-11-11T07:23:25,188 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/d418fef56007427a84da88b378c9ab69 2024-11-11T07:23:25,192 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into 8dbe8f01f71640d3a7f7bc892eabbad2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:25,192 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:25,192 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=13, startTime=1731309804327; duration=0sec 2024-11-11T07:23:25,192 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:25,193 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/7bfcb6b5efac465b8398640bef517c9c is 50, key is test_row_0/C:col10/1731309803074/Put/seqid=0 2024-11-11T07:23:25,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742113_1289 (size=12151) 2024-11-11T07:23:25,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309865222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309865222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309865222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,223 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,223 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309865222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,224 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,224 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309865331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,601 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/7bfcb6b5efac465b8398640bef517c9c 2024-11-11T07:23:25,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/7591de80bbb147848a626ebd33afc65b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/7591de80bbb147848a626ebd33afc65b 2024-11-11T07:23:25,612 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/7591de80bbb147848a626ebd33afc65b, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T07:23:25,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/d418fef56007427a84da88b378c9ab69 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/d418fef56007427a84da88b378c9ab69 2024-11-11T07:23:25,618 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/d418fef56007427a84da88b378c9ab69, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T07:23:25,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/7bfcb6b5efac465b8398640bef517c9c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/7bfcb6b5efac465b8398640bef517c9c 2024-11-11T07:23:25,624 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/7bfcb6b5efac465b8398640bef517c9c, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T07:23:25,625 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1272ms, sequenceid=248, compaction requested=false 2024-11-11T07:23:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:25,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:25,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T07:23:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T07:23:25,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T07:23:25,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6530 sec 2024-11-11T07:23:25,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.6580 sec 2024-11-11T07:23:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:25,846 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:25,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-11T07:23:25,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/6be7f52761464589b155453fac418955 is 50, key is test_row_0/A:col10/1731309805844/Put/seqid=0 2024-11-11T07:23:25,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742114_1290 (size=14741) 2024-11-11T07:23:25,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309865872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309865981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:26,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309866191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:26,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/6be7f52761464589b155453fac418955 2024-11-11T07:23:26,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/855de3c368d84dcc8fb730917e510613 is 50, key is test_row_0/B:col10/1731309805844/Put/seqid=0 2024-11-11T07:23:26,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742115_1291 (size=12301) 2024-11-11T07:23:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309866501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:26,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/855de3c368d84dcc8fb730917e510613 2024-11-11T07:23:26,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/3b5b32939f3845388d725c0f85478d00 is 50, key is test_row_0/C:col10/1731309805844/Put/seqid=0 2024-11-11T07:23:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742116_1292 (size=12301) 2024-11-11T07:23:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309867012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:23:27,142 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:27,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T07:23:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:23:27,148 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:27,149 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:27,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:27,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/3b5b32939f3845388d725c0f85478d00 2024-11-11T07:23:27,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/6be7f52761464589b155453fac418955 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6be7f52761464589b155453fac418955 2024-11-11T07:23:27,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6be7f52761464589b155453fac418955, entries=200, sequenceid=279, filesize=14.4 K 2024-11-11T07:23:27,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/855de3c368d84dcc8fb730917e510613 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/855de3c368d84dcc8fb730917e510613 2024-11-11T07:23:27,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/855de3c368d84dcc8fb730917e510613, entries=150, sequenceid=279, filesize=12.0 K 2024-11-11T07:23:27,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/3b5b32939f3845388d725c0f85478d00 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/3b5b32939f3845388d725c0f85478d00 2024-11-11T07:23:27,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/3b5b32939f3845388d725c0f85478d00, entries=150, sequenceid=279, filesize=12.0 K 2024-11-11T07:23:27,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1359ms, sequenceid=279, compaction requested=true 2024-11-11T07:23:27,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:27,206 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:27,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:27,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:27,212 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:27,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:27,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:27,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:27,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:27,214 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:27,214 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:27,214 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,215 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/aba80b83afa44b7dbd30cc36480492c3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/7591de80bbb147848a626ebd33afc65b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6be7f52761464589b155453fac418955] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=38.7 K 2024-11-11T07:23:27,216 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting aba80b83afa44b7dbd30cc36480492c3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731309802431 2024-11-11T07:23:27,216 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7591de80bbb147848a626ebd33afc65b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731309803069 2024-11-11T07:23:27,221 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be7f52761464589b155453fac418955, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731309804691 2024-11-11T07:23:27,222 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:27,223 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:27,223 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,223 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/9a7628f3f4f240a88abd2c1df63419eb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/d418fef56007427a84da88b378c9ab69, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/855de3c368d84dcc8fb730917e510613] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=36.3 K 2024-11-11T07:23:27,223 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a7628f3f4f240a88abd2c1df63419eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731309802431 2024-11-11T07:23:27,224 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting d418fef56007427a84da88b378c9ab69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731309803069 2024-11-11T07:23:27,226 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 855de3c368d84dcc8fb730917e510613, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731309804697 2024-11-11T07:23:27,244 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:27,245 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/bf246bc55a9e41ddb1db0a7507c54e81 is 50, key is test_row_0/A:col10/1731309805844/Put/seqid=0 2024-11-11T07:23:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:23:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:27,247 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:27,248 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/f4c5508c64cb4e33b7bf4cc9f5aa9f80 is 50, key is test_row_0/B:col10/1731309805844/Put/seqid=0 2024-11-11T07:23:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:23:27,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/e20cfde2324742bbb19145a105e3f7f8 is 50, key is test_row_0/A:col10/1731309807239/Put/seqid=0 2024-11-11T07:23:27,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309867298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309867300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309867300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309867305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742118_1294 (size=12949) 2024-11-11T07:23:27,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742117_1293 (size=12949) 2024-11-11T07:23:27,352 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/bf246bc55a9e41ddb1db0a7507c54e81 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bf246bc55a9e41ddb1db0a7507c54e81 2024-11-11T07:23:27,367 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into bf246bc55a9e41ddb1db0a7507c54e81(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:27,367 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:27,367 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=13, startTime=1731309807205; duration=0sec 2024-11-11T07:23:27,368 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:27,368 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:27,368 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:27,369 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:27,369 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:27,369 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,369 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8dbe8f01f71640d3a7f7bc892eabbad2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/7bfcb6b5efac465b8398640bef517c9c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/3b5b32939f3845388d725c0f85478d00] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=36.3 K 2024-11-11T07:23:27,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742119_1295 (size=9857) 2024-11-11T07:23:27,370 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbe8f01f71640d3a7f7bc892eabbad2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731309802431 2024-11-11T07:23:27,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/e20cfde2324742bbb19145a105e3f7f8 2024-11-11T07:23:27,371 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bfcb6b5efac465b8398640bef517c9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731309803069 2024-11-11T07:23:27,371 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5b32939f3845388d725c0f85478d00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731309804697 2024-11-11T07:23:27,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/cf634fb3907448d9bc0b38b32f6a889f is 50, key is test_row_0/B:col10/1731309807239/Put/seqid=0 2024-11-11T07:23:27,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309867412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309867413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,422 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:27,423 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/9b81f5f62d1a4ed18aec87e32e545a2f is 50, key is test_row_0/C:col10/1731309805844/Put/seqid=0 2024-11-11T07:23:27,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309867421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309867422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742120_1296 (size=9857) 2024-11-11T07:23:27,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:23:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742121_1297 (size=12949) 2024-11-11T07:23:27,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309867622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309867632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309867632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309867632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,757 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/f4c5508c64cb4e33b7bf4cc9f5aa9f80 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/f4c5508c64cb4e33b7bf4cc9f5aa9f80 2024-11-11T07:23:27,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:27,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:27,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:23:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,775 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into f4c5508c64cb4e33b7bf4cc9f5aa9f80(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:27,775 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:27,775 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=13, startTime=1731309807212; duration=0sec 2024-11-11T07:23:27,776 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:27,776 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:27,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/cf634fb3907448d9bc0b38b32f6a889f 2024-11-11T07:23:27,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2ce41c82f79546e1ab28d27a0a0b9c61 is 50, key is test_row_0/C:col10/1731309807239/Put/seqid=0 2024-11-11T07:23:27,890 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/9b81f5f62d1a4ed18aec87e32e545a2f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/9b81f5f62d1a4ed18aec87e32e545a2f 2024-11-11T07:23:27,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742122_1298 (size=9857) 2024-11-11T07:23:27,897 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into 9b81f5f62d1a4ed18aec87e32e545a2f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:27,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:27,897 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=13, startTime=1731309807213; duration=0sec 2024-11-11T07:23:27,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:27,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:27,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2ce41c82f79546e1ab28d27a0a0b9c61 2024-11-11T07:23:27,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/e20cfde2324742bbb19145a105e3f7f8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e20cfde2324742bbb19145a105e3f7f8 2024-11-11T07:23:27,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e20cfde2324742bbb19145a105e3f7f8, entries=100, sequenceid=290, filesize=9.6 K 2024-11-11T07:23:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/cf634fb3907448d9bc0b38b32f6a889f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/cf634fb3907448d9bc0b38b32f6a889f 2024-11-11T07:23:27,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:27,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:27,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:27,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:27,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309867932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/cf634fb3907448d9bc0b38b32f6a889f, entries=100, sequenceid=290, filesize=9.6 K 2024-11-11T07:23:27,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2ce41c82f79546e1ab28d27a0a0b9c61 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2ce41c82f79546e1ab28d27a0a0b9c61 2024-11-11T07:23:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309867942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309867942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2ce41c82f79546e1ab28d27a0a0b9c61, entries=100, sequenceid=290, filesize=9.6 K 2024-11-11T07:23:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309867942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:27,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c49bcd3ff91d850f5acc98ee12d10ac3 in 700ms, sequenceid=290, compaction requested=false 2024-11-11T07:23:27,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:28,036 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:28,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T07:23:28,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/12cd6ae7550e427b8ba33a3bbe627ab2 is 50, key is test_row_0/A:col10/1731309808033/Put/seqid=0 2024-11-11T07:23:28,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309868064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742123_1299 (size=14741) 2024-11-11T07:23:28,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309868172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:23:28,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309868381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309868442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309868451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309868451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309868461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/12cd6ae7550e427b8ba33a3bbe627ab2 2024-11-11T07:23:28,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/3ac3b90a7f344d138462b4c4c8feeda0 is 50, key is test_row_0/B:col10/1731309808033/Put/seqid=0 2024-11-11T07:23:28,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742124_1300 (size=12301) 2024-11-11T07:23:28,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/3ac3b90a7f344d138462b4c4c8feeda0 2024-11-11T07:23:28,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2c5539851f384a7c8e7cfeda43ad42f5 is 50, key is test_row_0/C:col10/1731309808033/Put/seqid=0 2024-11-11T07:23:28,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742125_1301 (size=12301) 2024-11-11T07:23:28,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2c5539851f384a7c8e7cfeda43ad42f5 2024-11-11T07:23:28,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/12cd6ae7550e427b8ba33a3bbe627ab2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12cd6ae7550e427b8ba33a3bbe627ab2 2024-11-11T07:23:28,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12cd6ae7550e427b8ba33a3bbe627ab2, entries=200, sequenceid=319, filesize=14.4 K 2024-11-11T07:23:28,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/3ac3b90a7f344d138462b4c4c8feeda0 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3ac3b90a7f344d138462b4c4c8feeda0 2024-11-11T07:23:28,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3ac3b90a7f344d138462b4c4c8feeda0, entries=150, sequenceid=319, filesize=12.0 K 2024-11-11T07:23:28,619 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-11T07:23:28,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/2c5539851f384a7c8e7cfeda43ad42f5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2c5539851f384a7c8e7cfeda43ad42f5 2024-11-11T07:23:28,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2c5539851f384a7c8e7cfeda43ad42f5, entries=150, sequenceid=319, filesize=12.0 K 2024-11-11T07:23:28,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c49bcd3ff91d850f5acc98ee12d10ac3 in 590ms, sequenceid=319, compaction requested=true 2024-11-11T07:23:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:28,626 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:28,626 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:28,627 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:28,627 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:28,627 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,627 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bf246bc55a9e41ddb1db0a7507c54e81, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e20cfde2324742bbb19145a105e3f7f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12cd6ae7550e427b8ba33a3bbe627ab2] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=36.7 K 2024-11-11T07:23:28,628 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:28,628 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:28,628 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,628 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/f4c5508c64cb4e33b7bf4cc9f5aa9f80, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/cf634fb3907448d9bc0b38b32f6a889f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3ac3b90a7f344d138462b4c4c8feeda0] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=34.3 K 2024-11-11T07:23:28,628 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf246bc55a9e41ddb1db0a7507c54e81, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731309804697 2024-11-11T07:23:28,629 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c5508c64cb4e33b7bf4cc9f5aa9f80, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731309804697 2024-11-11T07:23:28,629 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20cfde2324742bbb19145a105e3f7f8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731309807239 2024-11-11T07:23:28,629 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cf634fb3907448d9bc0b38b32f6a889f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731309807239 2024-11-11T07:23:28,629 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12cd6ae7550e427b8ba33a3bbe627ab2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731309807291 2024-11-11T07:23:28,630 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac3b90a7f344d138462b4c4c8feeda0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731309807296 2024-11-11T07:23:28,639 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:28,640 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/1edf52c82b9742bbac94b2b234370e5a is 50, key is test_row_0/A:col10/1731309808033/Put/seqid=0 2024-11-11T07:23:28,642 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:28,643 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/e9082f98782b4da1b4b9314bf13cfc63 is 50, key is test_row_0/B:col10/1731309808033/Put/seqid=0 2024-11-11T07:23:28,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742126_1302 (size=13051) 2024-11-11T07:23:28,653 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/1edf52c82b9742bbac94b2b234370e5a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/1edf52c82b9742bbac94b2b234370e5a 2024-11-11T07:23:28,661 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into 1edf52c82b9742bbac94b2b234370e5a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:28,661 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:28,661 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=13, startTime=1731309808626; duration=0sec 2024-11-11T07:23:28,661 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:28,661 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:28,661 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:28,663 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:28,663 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:28,663 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,663 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/9b81f5f62d1a4ed18aec87e32e545a2f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2ce41c82f79546e1ab28d27a0a0b9c61, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2c5539851f384a7c8e7cfeda43ad42f5] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=34.3 K 2024-11-11T07:23:28,664 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b81f5f62d1a4ed18aec87e32e545a2f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731309804697 2024-11-11T07:23:28,665 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ce41c82f79546e1ab28d27a0a0b9c61, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731309807239 2024-11-11T07:23:28,665 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c5539851f384a7c8e7cfeda43ad42f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731309807296 2024-11-11T07:23:28,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742127_1303 (size=13051) 2024-11-11T07:23:28,678 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/e9082f98782b4da1b4b9314bf13cfc63 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/e9082f98782b4da1b4b9314bf13cfc63 2024-11-11T07:23:28,686 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:28,686 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into e9082f98782b4da1b4b9314bf13cfc63(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:28,687 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:28,687 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=13, startTime=1731309808626; duration=0sec 2024-11-11T07:23:28,687 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:28,687 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:28,687 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/e6df701eab9e47e1b4ff224657dd4d36 is 50, key is test_row_0/C:col10/1731309808033/Put/seqid=0 2024-11-11T07:23:28,694 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:28,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:23:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:28,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:28,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:28,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/ee6c2487f26a4e85baa513db5b4ae774 is 50, key is test_row_0/A:col10/1731309808058/Put/seqid=0 2024-11-11T07:23:28,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742128_1304 (size=13051) 2024-11-11T07:23:28,755 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/e6df701eab9e47e1b4ff224657dd4d36 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/e6df701eab9e47e1b4ff224657dd4d36 2024-11-11T07:23:28,766 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into e6df701eab9e47e1b4ff224657dd4d36(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:28,767 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:28,767 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=13, startTime=1731309808626; duration=0sec 2024-11-11T07:23:28,767 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:28,767 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:28,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742129_1305 (size=12301) 2024-11-11T07:23:28,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:28,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309868874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:28,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309868981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:29,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:29,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:29,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:29,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:29,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309869191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:29,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/ee6c2487f26a4e85baa513db5b4ae774 2024-11-11T07:23:29,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d is 50, key is test_row_0/B:col10/1731309808058/Put/seqid=0 2024-11-11T07:23:29,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742130_1306 (size=12301) 2024-11-11T07:23:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:23:29,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:29,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309869451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:29,453 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:29,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309869461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:29,465 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:29,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309869472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:29,474 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:29,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309869482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:29,485 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:29,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309869501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:29,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d 2024-11-11T07:23:29,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/f3a0a329353e437b9b89568b3941567f is 50, key is test_row_0/C:col10/1731309808058/Put/seqid=0 2024-11-11T07:23:29,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742131_1307 (size=12301) 2024-11-11T07:23:29,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:29,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:29,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:30,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309870011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:30,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:30,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:30,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/f3a0a329353e437b9b89568b3941567f 2024-11-11T07:23:30,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/ee6c2487f26a4e85baa513db5b4ae774 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/ee6c2487f26a4e85baa513db5b4ae774 2024-11-11T07:23:30,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/ee6c2487f26a4e85baa513db5b4ae774, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T07:23:30,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d 2024-11-11T07:23:30,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T07:23:30,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/f3a0a329353e437b9b89568b3941567f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f3a0a329353e437b9b89568b3941567f 2024-11-11T07:23:30,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f3a0a329353e437b9b89568b3941567f, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T07:23:30,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1418ms, sequenceid=332, compaction requested=false 2024-11-11T07:23:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:30,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:23:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:30,238 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:23:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/6e1745ed34ae425da18eddd2a1a79b3f is 50, key is test_row_0/A:col10/1731309808841/Put/seqid=0 2024-11-11T07:23:30,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742132_1308 (size=12301) 2024-11-11T07:23:30,673 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/6e1745ed34ae425da18eddd2a1a79b3f 2024-11-11T07:23:30,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/a4c5ba80319a4245abe67c826f8d7ea4 is 50, key is test_row_0/B:col10/1731309808841/Put/seqid=0 2024-11-11T07:23:30,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742133_1309 (size=12301) 2024-11-11T07:23:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:31,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309871062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,092 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/a4c5ba80319a4245abe67c826f8d7ea4 2024-11-11T07:23:31,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/802c18089e1149eb878716258be7e888 is 50, key is test_row_0/C:col10/1731309808841/Put/seqid=0 2024-11-11T07:23:31,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742134_1310 (size=12301) 2024-11-11T07:23:31,133 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/802c18089e1149eb878716258be7e888 2024-11-11T07:23:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/6e1745ed34ae425da18eddd2a1a79b3f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6e1745ed34ae425da18eddd2a1a79b3f 2024-11-11T07:23:31,142 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6e1745ed34ae425da18eddd2a1a79b3f, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T07:23:31,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/a4c5ba80319a4245abe67c826f8d7ea4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/a4c5ba80319a4245abe67c826f8d7ea4 2024-11-11T07:23:31,151 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/a4c5ba80319a4245abe67c826f8d7ea4, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T07:23:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/802c18089e1149eb878716258be7e888 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/802c18089e1149eb878716258be7e888 2024-11-11T07:23:31,160 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/802c18089e1149eb878716258be7e888, entries=150, sequenceid=358, filesize=12.0 K 2024-11-11T07:23:31,162 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c49bcd3ff91d850f5acc98ee12d10ac3 in 923ms, sequenceid=358, compaction requested=true 2024-11-11T07:23:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T07:23:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T07:23:31,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T07:23:31,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0140 sec 2024-11-11T07:23:31,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 4.0240 sec 2024-11-11T07:23:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:31,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:31,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:23:31,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/f023ecf6327d4d2e86fe0442f829043b is 50, key is test_row_0/A:col10/1731309811054/Put/seqid=0 2024-11-11T07:23:31,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742135_1311 (size=14741) 2024-11-11T07:23:31,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/f023ecf6327d4d2e86fe0442f829043b 2024-11-11T07:23:31,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/96000f7c12a948d88e346e4abec4a789 is 50, key is test_row_0/B:col10/1731309811054/Put/seqid=0 2024-11-11T07:23:31,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742136_1312 (size=12301) 2024-11-11T07:23:31,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/96000f7c12a948d88e346e4abec4a789 2024-11-11T07:23:31,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/408d311c96dd4078a213c993bc2c8713 is 50, key is test_row_0/C:col10/1731309811054/Put/seqid=0 2024-11-11T07:23:31,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742137_1313 (size=12301) 2024-11-11T07:23:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:23:31,302 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:31,302 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T07:23:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:23:31,305 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:31,306 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:31,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:31,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309871321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:23:31,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309871432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:23:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:31,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1731309871462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,467 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36638 deadline: 1731309871491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36648 deadline: 1731309871492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,497 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,498 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36592 deadline: 1731309871511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,516 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in c49bcd3ff91d850f5acc98ee12d10ac3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:23:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:23:31,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309871641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:31,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/408d311c96dd4078a213c993bc2c8713 2024-11-11T07:23:31,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/f023ecf6327d4d2e86fe0442f829043b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/f023ecf6327d4d2e86fe0442f829043b 2024-11-11T07:23:31,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/f023ecf6327d4d2e86fe0442f829043b, entries=200, sequenceid=370, filesize=14.4 K 2024-11-11T07:23:31,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/96000f7c12a948d88e346e4abec4a789 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/96000f7c12a948d88e346e4abec4a789 2024-11-11T07:23:31,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/96000f7c12a948d88e346e4abec4a789, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T07:23:31,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/408d311c96dd4078a213c993bc2c8713 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/408d311c96dd4078a213c993bc2c8713 2024-11-11T07:23:31,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/408d311c96dd4078a213c993bc2c8713, entries=150, sequenceid=370, filesize=12.0 K 2024-11-11T07:23:31,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c49bcd3ff91d850f5acc98ee12d10ac3 in 501ms, sequenceid=370, compaction requested=true 2024-11-11T07:23:31,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:31,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:31,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:31,681 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:31,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:31,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:31,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:31,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:23:31,681 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:31,683 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:31,683 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:31,683 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:31,683 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,683 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:31,683 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/e9082f98782b4da1b4b9314bf13cfc63, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/a4c5ba80319a4245abe67c826f8d7ea4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/96000f7c12a948d88e346e4abec4a789] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=48.8 K 2024-11-11T07:23:31,683 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,683 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/1edf52c82b9742bbac94b2b234370e5a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/ee6c2487f26a4e85baa513db5b4ae774, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6e1745ed34ae425da18eddd2a1a79b3f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/f023ecf6327d4d2e86fe0442f829043b] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=51.2 K 2024-11-11T07:23:31,683 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e9082f98782b4da1b4b9314bf13cfc63, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731309807296 2024-11-11T07:23:31,684 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1edf52c82b9742bbac94b2b234370e5a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731309807296 2024-11-11T07:23:31,684 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fcc86a6b7c64fc68c2b3adce4bf4a9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731309808050 2024-11-11T07:23:31,684 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee6c2487f26a4e85baa513db5b4ae774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731309808050 2024-11-11T07:23:31,685 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1745ed34ae425da18eddd2a1a79b3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731309808813 2024-11-11T07:23:31,685 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c5ba80319a4245abe67c826f8d7ea4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731309808813 2024-11-11T07:23:31,685 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting f023ecf6327d4d2e86fe0442f829043b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731309811028 2024-11-11T07:23:31,686 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 96000f7c12a948d88e346e4abec4a789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731309811054 2024-11-11T07:23:31,698 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#272 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:31,698 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#273 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:31,698 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/c189996058504bc0a1eaa0e7fa07929f is 50, key is test_row_0/B:col10/1731309811054/Put/seqid=0 2024-11-11T07:23:31,698 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/62c9a08d6da0439ca3ffbf3cd5a6aac2 is 50, key is test_row_0/A:col10/1731309811054/Put/seqid=0 2024-11-11T07:23:31,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742138_1314 (size=13187) 2024-11-11T07:23:31,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742139_1315 (size=13187) 2024-11-11T07:23:31,729 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/c189996058504bc0a1eaa0e7fa07929f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/c189996058504bc0a1eaa0e7fa07929f 2024-11-11T07:23:31,731 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/62c9a08d6da0439ca3ffbf3cd5a6aac2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/62c9a08d6da0439ca3ffbf3cd5a6aac2 2024-11-11T07:23:31,737 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into c189996058504bc0a1eaa0e7fa07929f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:31,737 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:31,737 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=12, startTime=1731309811681; duration=0sec 2024-11-11T07:23:31,737 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:31,737 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:31,737 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:31,738 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into 62c9a08d6da0439ca3ffbf3cd5a6aac2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:31,738 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:31,739 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=12, startTime=1731309811681; duration=0sec 2024-11-11T07:23:31,739 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:31,739 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:31,739 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:31,739 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:31,740 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,740 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/e6df701eab9e47e1b4ff224657dd4d36, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f3a0a329353e437b9b89568b3941567f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/802c18089e1149eb878716258be7e888, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/408d311c96dd4078a213c993bc2c8713] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=48.8 K 2024-11-11T07:23:31,740 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e6df701eab9e47e1b4ff224657dd4d36, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731309807296 2024-11-11T07:23:31,741 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a0a329353e437b9b89568b3941567f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731309808050 2024-11-11T07:23:31,741 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 802c18089e1149eb878716258be7e888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731309808813 2024-11-11T07:23:31,741 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 408d311c96dd4078a213c993bc2c8713, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731309811054 2024-11-11T07:23:31,751 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:31,751 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/957561514f2e4869bc9ca07ce65b32c0 is 50, key is test_row_0/C:col10/1731309811054/Put/seqid=0 2024-11-11T07:23:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742140_1316 (size=13187) 2024-11-11T07:23:31,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:23:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:31,771 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:23:31,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2aaa868c9e654e5fbcd741c84ed5c396 is 50, key is test_row_0/A:col10/1731309811313/Put/seqid=0 2024-11-11T07:23:31,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742141_1317 (size=12301) 2024-11-11T07:23:31,784 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2aaa868c9e654e5fbcd741c84ed5c396 2024-11-11T07:23:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/8972edf4576e4e22a68f96bd66cd0854 is 50, key is test_row_0/B:col10/1731309811313/Put/seqid=0 2024-11-11T07:23:31,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742142_1318 (size=12301) 2024-11-11T07:23:31,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:23:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:31,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:32,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309871997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:32,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309872112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:32,162 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/957561514f2e4869bc9ca07ce65b32c0 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/957561514f2e4869bc9ca07ce65b32c0 2024-11-11T07:23:32,170 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into 957561514f2e4869bc9ca07ce65b32c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:32,170 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:32,170 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=12, startTime=1731309811681; duration=0sec 2024-11-11T07:23:32,170 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:32,170 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:32,212 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/8972edf4576e4e22a68f96bd66cd0854 2024-11-11T07:23:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/997bbfd16ba141f8b9473f7b5d073d8b is 50, key is test_row_0/C:col10/1731309811313/Put/seqid=0 2024-11-11T07:23:32,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742143_1319 (size=12301) 2024-11-11T07:23:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309872321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:23:32,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309872632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:32,653 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/997bbfd16ba141f8b9473f7b5d073d8b 2024-11-11T07:23:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/2aaa868c9e654e5fbcd741c84ed5c396 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2aaa868c9e654e5fbcd741c84ed5c396 2024-11-11T07:23:32,675 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2aaa868c9e654e5fbcd741c84ed5c396, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T07:23:32,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/8972edf4576e4e22a68f96bd66cd0854 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8972edf4576e4e22a68f96bd66cd0854 2024-11-11T07:23:32,683 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8972edf4576e4e22a68f96bd66cd0854, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T07:23:32,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/997bbfd16ba141f8b9473f7b5d073d8b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/997bbfd16ba141f8b9473f7b5d073d8b 2024-11-11T07:23:32,689 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/997bbfd16ba141f8b9473f7b5d073d8b, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T07:23:32,698 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c49bcd3ff91d850f5acc98ee12d10ac3 in 927ms, sequenceid=396, compaction requested=false 2024-11-11T07:23:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T07:23:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T07:23:32,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T07:23:32,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3940 sec 2024-11-11T07:23:32,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4020 sec 2024-11-11T07:23:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:33,145 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:33,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:23:33,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/9b91d0334fde4dfb9b46959a5809b47f is 50, key is test_row_0/A:col10/1731309813144/Put/seqid=0 2024-11-11T07:23:33,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742144_1320 (size=14741) 2024-11-11T07:23:33,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/9b91d0334fde4dfb9b46959a5809b47f 2024-11-11T07:23:33,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/382e90f468cd40b388ea1919ae84c0d7 is 50, key is test_row_0/B:col10/1731309813144/Put/seqid=0 2024-11-11T07:23:33,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742145_1321 (size=12301) 2024-11-11T07:23:33,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309873270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309873382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:23:33,452 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:33,452 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T07:23:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:23:33,455 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:33,456 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:33,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:23:33,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309873592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:33,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/382e90f468cd40b388ea1919ae84c0d7 2024-11-11T07:23:33,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:23:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:33,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/f10a636281334451bba2a35f8178ab5c is 50, key is test_row_0/C:col10/1731309813144/Put/seqid=0 2024-11-11T07:23:33,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742146_1322 (size=12301) 2024-11-11T07:23:33,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:23:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:33,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:23:33,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309873902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:33,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:23:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:33,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:34,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/f10a636281334451bba2a35f8178ab5c 2024-11-11T07:23:34,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/9b91d0334fde4dfb9b46959a5809b47f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/9b91d0334fde4dfb9b46959a5809b47f 2024-11-11T07:23:34,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/9b91d0334fde4dfb9b46959a5809b47f, entries=200, sequenceid=410, filesize=14.4 K 2024-11-11T07:23:34,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/382e90f468cd40b388ea1919ae84c0d7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/382e90f468cd40b388ea1919ae84c0d7 2024-11-11T07:23:34,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/382e90f468cd40b388ea1919ae84c0d7, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T07:23:34,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/f10a636281334451bba2a35f8178ab5c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f10a636281334451bba2a35f8178ab5c 2024-11-11T07:23:34,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f10a636281334451bba2a35f8178ab5c, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T07:23:34,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c49bcd3ff91d850f5acc98ee12d10ac3 in 903ms, sequenceid=410, compaction requested=true 2024-11-11T07:23:34,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:34,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:34,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:34,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:34,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:34,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:34,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:34,049 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:34,049 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:34,049 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:34,049 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:34,049 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/B is initiating minor compaction (all files) 2024-11-11T07:23:34,050 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/A is initiating minor compaction (all files) 2024-11-11T07:23:34,050 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/A in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:34,050 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/62c9a08d6da0439ca3ffbf3cd5a6aac2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2aaa868c9e654e5fbcd741c84ed5c396, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/9b91d0334fde4dfb9b46959a5809b47f] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=39.3 K 2024-11-11T07:23:34,050 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/B in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:34,050 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/c189996058504bc0a1eaa0e7fa07929f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8972edf4576e4e22a68f96bd66cd0854, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/382e90f468cd40b388ea1919ae84c0d7] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=36.9 K 2024-11-11T07:23:34,051 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c189996058504bc0a1eaa0e7fa07929f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731309811054 2024-11-11T07:23:34,051 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c9a08d6da0439ca3ffbf3cd5a6aac2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731309811054 2024-11-11T07:23:34,051 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8972edf4576e4e22a68f96bd66cd0854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731309811301 2024-11-11T07:23:34,051 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aaa868c9e654e5fbcd741c84ed5c396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731309811301 2024-11-11T07:23:34,052 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 382e90f468cd40b388ea1919ae84c0d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731309811990 2024-11-11T07:23:34,052 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b91d0334fde4dfb9b46959a5809b47f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731309811977 2024-11-11T07:23:34,059 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#B#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:34,060 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/69f0e13fbdfb4e96a95f2ab7ea28b5af is 50, key is test_row_0/B:col10/1731309813144/Put/seqid=0 2024-11-11T07:23:34,063 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:34,064 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/d4022378f1cf4094a087d1be671d2fd4 is 50, key is test_row_0/A:col10/1731309813144/Put/seqid=0 2024-11-11T07:23:34,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:23:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:34,070 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:23:34,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742147_1323 (size=13289) 2024-11-11T07:23:34,080 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/69f0e13fbdfb4e96a95f2ab7ea28b5af as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/69f0e13fbdfb4e96a95f2ab7ea28b5af 2024-11-11T07:23:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:23:34,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742148_1324 (size=13289) 2024-11-11T07:23:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/a87bc0eb113f4738b865ac8baf25ae46 is 50, key is test_row_0/A:col10/1731309813264/Put/seqid=0 2024-11-11T07:23:34,087 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/B of c49bcd3ff91d850f5acc98ee12d10ac3 into 69f0e13fbdfb4e96a95f2ab7ea28b5af(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:34,087 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:34,087 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/B, priority=13, startTime=1731309814049; duration=0sec 2024-11-11T07:23:34,088 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:34,088 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:34,088 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:34,089 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:34,090 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): c49bcd3ff91d850f5acc98ee12d10ac3/C is initiating minor compaction (all files) 2024-11-11T07:23:34,090 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49bcd3ff91d850f5acc98ee12d10ac3/C in TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:34,090 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/957561514f2e4869bc9ca07ce65b32c0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/997bbfd16ba141f8b9473f7b5d073d8b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f10a636281334451bba2a35f8178ab5c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp, totalSize=36.9 K 2024-11-11T07:23:34,090 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/d4022378f1cf4094a087d1be671d2fd4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d4022378f1cf4094a087d1be671d2fd4 2024-11-11T07:23:34,090 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 957561514f2e4869bc9ca07ce65b32c0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731309811054 2024-11-11T07:23:34,090 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 997bbfd16ba141f8b9473f7b5d073d8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731309811301 2024-11-11T07:23:34,091 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f10a636281334451bba2a35f8178ab5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731309811990 2024-11-11T07:23:34,096 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/A of c49bcd3ff91d850f5acc98ee12d10ac3 into d4022378f1cf4094a087d1be671d2fd4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:34,096 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:34,096 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/A, priority=13, startTime=1731309814048; duration=0sec 2024-11-11T07:23:34,096 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:34,096 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:34,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742149_1325 (size=12301) 2024-11-11T07:23:34,122 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49bcd3ff91d850f5acc98ee12d10ac3#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:34,123 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/de8593d891ae4c30bbfa377bd9400ef8 is 50, key is test_row_0/C:col10/1731309813144/Put/seqid=0 2024-11-11T07:23:34,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742150_1326 (size=13289) 2024-11-11T07:23:34,174 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/de8593d891ae4c30bbfa377bd9400ef8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/de8593d891ae4c30bbfa377bd9400ef8 2024-11-11T07:23:34,180 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49bcd3ff91d850f5acc98ee12d10ac3/C of c49bcd3ff91d850f5acc98ee12d10ac3 into de8593d891ae4c30bbfa377bd9400ef8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:34,180 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:34,180 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., storeName=c49bcd3ff91d850f5acc98ee12d10ac3/C, priority=13, startTime=1731309814049; duration=0sec 2024-11-11T07:23:34,180 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:34,181 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:34,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. as already flushing 2024-11-11T07:23:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309874451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:34,514 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/a87bc0eb113f4738b865ac8baf25ae46 2024-11-11T07:23:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/ff0281d229c14734bb719da058d19dec is 50, key is test_row_0/B:col10/1731309813264/Put/seqid=0 2024-11-11T07:23:34,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742151_1327 (size=12301) 2024-11-11T07:23:34,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309874562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:23:34,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309874772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:34,944 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/ff0281d229c14734bb719da058d19dec 2024-11-11T07:23:34,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/ca58dae9b2f94cde8a9d1878769a8b56 is 50, key is test_row_0/C:col10/1731309813264/Put/seqid=0 2024-11-11T07:23:34,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742152_1328 (size=12301) 2024-11-11T07:23:35,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731309875081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:23:35,166 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-11T07:23:35,166 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,166 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,166 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,169 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-11T07:23:35,169 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,169 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,170 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-11T07:23:35,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,170 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,170 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,170 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,170 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,170 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-11T07:23:35,170 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,170 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,171 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,172 INFO [Thread-1116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1116. 2024-11-11T07:23:35,173 DEBUG [Thread-1116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,173 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,173 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,366 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/ca58dae9b2f94cde8a9d1878769a8b56 2024-11-11T07:23:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/a87bc0eb113f4738b865ac8baf25ae46 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/a87bc0eb113f4738b865ac8baf25ae46 2024-11-11T07:23:35,374 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/a87bc0eb113f4738b865ac8baf25ae46, entries=150, sequenceid=433, filesize=12.0 K 2024-11-11T07:23:35,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/ff0281d229c14734bb719da058d19dec as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/ff0281d229c14734bb719da058d19dec 2024-11-11T07:23:35,377 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/ff0281d229c14734bb719da058d19dec, entries=150, sequenceid=433, filesize=12.0 K 2024-11-11T07:23:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/ca58dae9b2f94cde8a9d1878769a8b56 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ca58dae9b2f94cde8a9d1878769a8b56 2024-11-11T07:23:35,381 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ca58dae9b2f94cde8a9d1878769a8b56, entries=150, sequenceid=433, filesize=12.0 K 2024-11-11T07:23:35,382 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1312ms, sequenceid=433, compaction requested=false 2024-11-11T07:23:35,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:35,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:35,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T07:23:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T07:23:35,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T07:23:35,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9270 sec 2024-11-11T07:23:35,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.9330 sec 2024-11-11T07:23:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:35,483 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:35,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:23:35,483 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-11T07:23:35,483 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,483 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,483 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/8e040c6d429e4d8daf584586f5adc18f is 50, key is test_row_0/A:col10/1731309814450/Put/seqid=0 2024-11-11T07:23:35,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742153_1329 (size=12301) 2024-11-11T07:23:35,513 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-11T07:23:35,513 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,513 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,514 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,522 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-11T07:23:35,522 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,522 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,523 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,543 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-11T07:23:35,543 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,543 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,543 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,593 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-11T07:23:35,593 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:23:35,593 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,593 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:23:35,602 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3342 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3348 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3313 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3335 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3298 2024-11-11T07:23:35,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:23:35,602 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:23:35,602 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:23:35,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:35,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:23:35,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:23:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:23:35,606 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309815606"}]},"ts":"1731309815606"} 2024-11-11T07:23:35,607 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:23:35,607 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:23:35,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:23:35,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, UNASSIGN}] 2024-11-11T07:23:35,610 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, UNASSIGN 2024-11-11T07:23:35,610 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=c49bcd3ff91d850f5acc98ee12d10ac3, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:35,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, UNASSIGN because future has completed 2024-11-11T07:23:35,611 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:23:35,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:23:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:23:35,764 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:35,764 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:23:35,764 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing c49bcd3ff91d850f5acc98ee12d10ac3, disabling compactions & flushes 2024-11-11T07:23:35,764 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:35,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/8e040c6d429e4d8daf584586f5adc18f 2024-11-11T07:23:35,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/be8dcf58924a45fab362b3c106734a2b is 50, key is test_row_0/B:col10/1731309814450/Put/seqid=0 2024-11-11T07:23:35,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742154_1330 (size=12301) 2024-11-11T07:23:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:23:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:23:36,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/be8dcf58924a45fab362b3c106734a2b 2024-11-11T07:23:36,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/939fbed019534fac9cf9e09f05f4b0ad is 50, key is test_row_0/C:col10/1731309814450/Put/seqid=0 2024-11-11T07:23:36,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742155_1331 (size=12301) 2024-11-11T07:23:36,528 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:23:36,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/939fbed019534fac9cf9e09f05f4b0ad 2024-11-11T07:23:36,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/8e040c6d429e4d8daf584586f5adc18f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/8e040c6d429e4d8daf584586f5adc18f 2024-11-11T07:23:36,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/8e040c6d429e4d8daf584586f5adc18f, entries=150, sequenceid=450, filesize=12.0 K 2024-11-11T07:23:36,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/be8dcf58924a45fab362b3c106734a2b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/be8dcf58924a45fab362b3c106734a2b 2024-11-11T07:23:36,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/be8dcf58924a45fab362b3c106734a2b, entries=150, sequenceid=450, filesize=12.0 K 2024-11-11T07:23:36,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/939fbed019534fac9cf9e09f05f4b0ad as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/939fbed019534fac9cf9e09f05f4b0ad 2024-11-11T07:23:36,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/939fbed019534fac9cf9e09f05f4b0ad, entries=150, sequenceid=450, filesize=12.0 K 2024-11-11T07:23:36,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=26.84 KB/27480 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1245ms, sequenceid=450, compaction requested=true 2024-11-11T07:23:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49bcd3ff91d850f5acc98ee12d10ac3: 2024-11-11T07:23:36,729 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:36,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. because compaction request was cancelled 2024-11-11T07:23:36,729 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:36,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:A 2024-11-11T07:23:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:36,729 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. after waiting 0 ms 2024-11-11T07:23:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:36,729 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:36,729 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. because compaction request was cancelled 2024-11-11T07:23:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49bcd3ff91d850f5acc98ee12d10ac3:C, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:36,729 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:B 2024-11-11T07:23:36,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. because compaction request was cancelled 2024-11-11T07:23:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:36,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49bcd3ff91d850f5acc98ee12d10ac3:C 2024-11-11T07:23:36,729 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing c49bcd3ff91d850f5acc98ee12d10ac3 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T07:23:36,733 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/4532012c19c941149e2681640051b6cf is 50, key is test_row_0/A:col10/1731309815512/Put/seqid=0 2024-11-11T07:23:36,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742156_1332 (size=12301) 2024-11-11T07:23:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:23:37,137 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/4532012c19c941149e2681640051b6cf 2024-11-11T07:23:37,144 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/583d868030fa460983c4cf0676232d44 is 50, key is test_row_0/B:col10/1731309815512/Put/seqid=0 2024-11-11T07:23:37,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742157_1333 (size=12301) 2024-11-11T07:23:37,547 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/583d868030fa460983c4cf0676232d44 2024-11-11T07:23:37,554 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/c234998036924635b72d06b9d3ad96ab is 50, key is test_row_0/C:col10/1731309815512/Put/seqid=0 2024-11-11T07:23:37,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742158_1334 (size=12301) 2024-11-11T07:23:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:23:37,959 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/c234998036924635b72d06b9d3ad96ab 2024-11-11T07:23:37,963 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/A/4532012c19c941149e2681640051b6cf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4532012c19c941149e2681640051b6cf 2024-11-11T07:23:37,966 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4532012c19c941149e2681640051b6cf, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T07:23:37,967 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/B/583d868030fa460983c4cf0676232d44 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/583d868030fa460983c4cf0676232d44 2024-11-11T07:23:37,970 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/583d868030fa460983c4cf0676232d44, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T07:23:37,970 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/.tmp/C/c234998036924635b72d06b9d3ad96ab as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/c234998036924635b72d06b9d3ad96ab 2024-11-11T07:23:37,973 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/c234998036924635b72d06b9d3ad96ab, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T07:23:37,974 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1245ms, sequenceid=457, compaction requested=true 2024-11-11T07:23:37,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/db6b5946003740349ed994c0cd1a63e7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/cbd0fdea1d094430bae205e09b41e86c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/967e0bfb9dde48c99cf342fc5b0973a4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2a5efd5bdaee480b80af2edc679dea16, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/415e1391edeb4d55b038b2ac558ff97a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4821a59d46944279aa6c98005c9669c6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/df054fb81fda4c02b62df550375b41b2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/5a8cc2a0305349579725f69e5b2c7503, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e072f45908104f65b1290a5293a12954, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12d61628b575416ca18ba04aa0e1beda, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2ee7edccb31045c0892061265376911c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/49d430eecdd848068f79f53dd2e87885, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d6e4d2c21e2c429ea5eb87edb2eb3842, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bcb1fad27dfe4dc9aa1799fc91a85e60, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/151751d3a04b453ab9cbd26afc09d0ce, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/36d9a18a843945e6ac7dbc607b941d8d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/aba80b83afa44b7dbd30cc36480492c3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/7591de80bbb147848a626ebd33afc65b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6be7f52761464589b155453fac418955, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bf246bc55a9e41ddb1db0a7507c54e81, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e20cfde2324742bbb19145a105e3f7f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12cd6ae7550e427b8ba33a3bbe627ab2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/1edf52c82b9742bbac94b2b234370e5a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/ee6c2487f26a4e85baa513db5b4ae774, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6e1745ed34ae425da18eddd2a1a79b3f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/f023ecf6327d4d2e86fe0442f829043b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/62c9a08d6da0439ca3ffbf3cd5a6aac2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2aaa868c9e654e5fbcd741c84ed5c396, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/9b91d0334fde4dfb9b46959a5809b47f] to archive 2024-11-11T07:23:37,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:23:37,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/db6b5946003740349ed994c0cd1a63e7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/db6b5946003740349ed994c0cd1a63e7 2024-11-11T07:23:37,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/cbd0fdea1d094430bae205e09b41e86c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/cbd0fdea1d094430bae205e09b41e86c 2024-11-11T07:23:37,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/967e0bfb9dde48c99cf342fc5b0973a4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/967e0bfb9dde48c99cf342fc5b0973a4 2024-11-11T07:23:37,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2a5efd5bdaee480b80af2edc679dea16 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2a5efd5bdaee480b80af2edc679dea16 2024-11-11T07:23:37,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/415e1391edeb4d55b038b2ac558ff97a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/415e1391edeb4d55b038b2ac558ff97a 2024-11-11T07:23:37,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4821a59d46944279aa6c98005c9669c6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4821a59d46944279aa6c98005c9669c6 2024-11-11T07:23:37,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/df054fb81fda4c02b62df550375b41b2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/df054fb81fda4c02b62df550375b41b2 2024-11-11T07:23:37,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/5a8cc2a0305349579725f69e5b2c7503 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/5a8cc2a0305349579725f69e5b2c7503 2024-11-11T07:23:37,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e072f45908104f65b1290a5293a12954 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e072f45908104f65b1290a5293a12954 2024-11-11T07:23:37,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12d61628b575416ca18ba04aa0e1beda to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12d61628b575416ca18ba04aa0e1beda 2024-11-11T07:23:37,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2ee7edccb31045c0892061265376911c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2ee7edccb31045c0892061265376911c 2024-11-11T07:23:37,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/49d430eecdd848068f79f53dd2e87885 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/49d430eecdd848068f79f53dd2e87885 2024-11-11T07:23:37,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d6e4d2c21e2c429ea5eb87edb2eb3842 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d6e4d2c21e2c429ea5eb87edb2eb3842 2024-11-11T07:23:37,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bcb1fad27dfe4dc9aa1799fc91a85e60 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bcb1fad27dfe4dc9aa1799fc91a85e60 2024-11-11T07:23:37,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/151751d3a04b453ab9cbd26afc09d0ce to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/151751d3a04b453ab9cbd26afc09d0ce 2024-11-11T07:23:37,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/36d9a18a843945e6ac7dbc607b941d8d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/36d9a18a843945e6ac7dbc607b941d8d 2024-11-11T07:23:37,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/aba80b83afa44b7dbd30cc36480492c3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/aba80b83afa44b7dbd30cc36480492c3 2024-11-11T07:23:37,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/7591de80bbb147848a626ebd33afc65b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/7591de80bbb147848a626ebd33afc65b 2024-11-11T07:23:37,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6be7f52761464589b155453fac418955 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6be7f52761464589b155453fac418955 2024-11-11T07:23:37,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bf246bc55a9e41ddb1db0a7507c54e81 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/bf246bc55a9e41ddb1db0a7507c54e81 2024-11-11T07:23:37,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e20cfde2324742bbb19145a105e3f7f8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/e20cfde2324742bbb19145a105e3f7f8 2024-11-11T07:23:37,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12cd6ae7550e427b8ba33a3bbe627ab2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/12cd6ae7550e427b8ba33a3bbe627ab2 2024-11-11T07:23:37,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/1edf52c82b9742bbac94b2b234370e5a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/1edf52c82b9742bbac94b2b234370e5a 2024-11-11T07:23:37,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/ee6c2487f26a4e85baa513db5b4ae774 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/ee6c2487f26a4e85baa513db5b4ae774 2024-11-11T07:23:37,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6e1745ed34ae425da18eddd2a1a79b3f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/6e1745ed34ae425da18eddd2a1a79b3f 2024-11-11T07:23:37,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/f023ecf6327d4d2e86fe0442f829043b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/f023ecf6327d4d2e86fe0442f829043b 2024-11-11T07:23:38,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/62c9a08d6da0439ca3ffbf3cd5a6aac2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/62c9a08d6da0439ca3ffbf3cd5a6aac2 2024-11-11T07:23:38,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2aaa868c9e654e5fbcd741c84ed5c396 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/2aaa868c9e654e5fbcd741c84ed5c396 2024-11-11T07:23:38,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/9b91d0334fde4dfb9b46959a5809b47f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/9b91d0334fde4dfb9b46959a5809b47f 2024-11-11T07:23:38,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/b166abe8b55a41b1af699004798c61d1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd32b9b3722448bebdc61dde3d635a77, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/919a4c780fb446fb884f0f34d595e595, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/704217775072467caa7151e5745ccb9a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/2fcc252179194fea8bf51fbbb7f1525b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fc151d0b476d4b02a1ecb2c8c2f8e1c3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/7661686c138743148896988a4f48f7b2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3b37ea73c86a470892a78444267bca18, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/25fc5eac0d194cddb721b778a6d01e54, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/748fa1eefe4f458fba3abf9d7d21d0ff, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/81ffc13ab6c74ef388df14482f44e405, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd0fd015f27c46a7b7ae453da7739df9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/82bdc00f773543659075ef42fdd12862, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8b5230ef61344c52bfe51a1b4d7b3d36, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/856b23cb6fa145f6981d282f9670afe2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/9a7628f3f4f240a88abd2c1df63419eb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/531e5afd94c04fd99b3b6d0eedd1c53a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/d418fef56007427a84da88b378c9ab69, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/f4c5508c64cb4e33b7bf4cc9f5aa9f80, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/855de3c368d84dcc8fb730917e510613, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/cf634fb3907448d9bc0b38b32f6a889f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/e9082f98782b4da1b4b9314bf13cfc63, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3ac3b90a7f344d138462b4c4c8feeda0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/a4c5ba80319a4245abe67c826f8d7ea4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/c189996058504bc0a1eaa0e7fa07929f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/96000f7c12a948d88e346e4abec4a789, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8972edf4576e4e22a68f96bd66cd0854, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/382e90f468cd40b388ea1919ae84c0d7] to archive 2024-11-11T07:23:38,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:23:38,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/b166abe8b55a41b1af699004798c61d1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/b166abe8b55a41b1af699004798c61d1 2024-11-11T07:23:38,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd32b9b3722448bebdc61dde3d635a77 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd32b9b3722448bebdc61dde3d635a77 2024-11-11T07:23:38,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/919a4c780fb446fb884f0f34d595e595 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/919a4c780fb446fb884f0f34d595e595 2024-11-11T07:23:38,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/704217775072467caa7151e5745ccb9a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/704217775072467caa7151e5745ccb9a 2024-11-11T07:23:38,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/2fcc252179194fea8bf51fbbb7f1525b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/2fcc252179194fea8bf51fbbb7f1525b 2024-11-11T07:23:38,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fc151d0b476d4b02a1ecb2c8c2f8e1c3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fc151d0b476d4b02a1ecb2c8c2f8e1c3 2024-11-11T07:23:38,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/7661686c138743148896988a4f48f7b2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/7661686c138743148896988a4f48f7b2 2024-11-11T07:23:38,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3b37ea73c86a470892a78444267bca18 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3b37ea73c86a470892a78444267bca18 2024-11-11T07:23:38,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/25fc5eac0d194cddb721b778a6d01e54 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/25fc5eac0d194cddb721b778a6d01e54 2024-11-11T07:23:38,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/748fa1eefe4f458fba3abf9d7d21d0ff to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/748fa1eefe4f458fba3abf9d7d21d0ff 2024-11-11T07:23:38,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/81ffc13ab6c74ef388df14482f44e405 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/81ffc13ab6c74ef388df14482f44e405 2024-11-11T07:23:38,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd0fd015f27c46a7b7ae453da7739df9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/fd0fd015f27c46a7b7ae453da7739df9 2024-11-11T07:23:38,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/82bdc00f773543659075ef42fdd12862 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/82bdc00f773543659075ef42fdd12862 2024-11-11T07:23:38,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8b5230ef61344c52bfe51a1b4d7b3d36 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8b5230ef61344c52bfe51a1b4d7b3d36 2024-11-11T07:23:38,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/856b23cb6fa145f6981d282f9670afe2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/856b23cb6fa145f6981d282f9670afe2 2024-11-11T07:23:38,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/9a7628f3f4f240a88abd2c1df63419eb to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/9a7628f3f4f240a88abd2c1df63419eb 2024-11-11T07:23:38,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/531e5afd94c04fd99b3b6d0eedd1c53a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/531e5afd94c04fd99b3b6d0eedd1c53a 2024-11-11T07:23:38,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/d418fef56007427a84da88b378c9ab69 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/d418fef56007427a84da88b378c9ab69 2024-11-11T07:23:38,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/f4c5508c64cb4e33b7bf4cc9f5aa9f80 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/f4c5508c64cb4e33b7bf4cc9f5aa9f80 2024-11-11T07:23:38,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/855de3c368d84dcc8fb730917e510613 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/855de3c368d84dcc8fb730917e510613 2024-11-11T07:23:38,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/cf634fb3907448d9bc0b38b32f6a889f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/cf634fb3907448d9bc0b38b32f6a889f 2024-11-11T07:23:38,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/e9082f98782b4da1b4b9314bf13cfc63 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/e9082f98782b4da1b4b9314bf13cfc63 2024-11-11T07:23:38,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3ac3b90a7f344d138462b4c4c8feeda0 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/3ac3b90a7f344d138462b4c4c8feeda0 2024-11-11T07:23:38,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/5fcc86a6b7c64fc68c2b3adce4bf4a9d 2024-11-11T07:23:38,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/a4c5ba80319a4245abe67c826f8d7ea4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/a4c5ba80319a4245abe67c826f8d7ea4 2024-11-11T07:23:38,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/c189996058504bc0a1eaa0e7fa07929f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/c189996058504bc0a1eaa0e7fa07929f 2024-11-11T07:23:38,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/96000f7c12a948d88e346e4abec4a789 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/96000f7c12a948d88e346e4abec4a789 2024-11-11T07:23:38,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8972edf4576e4e22a68f96bd66cd0854 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/8972edf4576e4e22a68f96bd66cd0854 2024-11-11T07:23:38,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/382e90f468cd40b388ea1919ae84c0d7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/382e90f468cd40b388ea1919ae84c0d7 2024-11-11T07:23:38,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0310d27720c54cc58a26c9561dfa8447, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8bb99cd4d4024bd6a06e99c478a5531e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/da96988635d148ae8f2387b07c08f9c3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/a1060117a66b4bfa983b62fadf4d37f4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2a3122ce187a4382bf09e0dd7b0e69a6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c20fa47525b4ab7b6faf7ff4da15d65, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/d5caed4041f44f1dad43b1825da54a68, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ee3b6469ade847b0a323bfd786dc7c78, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/bb4abb4f730943368434331a8201799b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c34832285d5424cbc42aad8144d0b67, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/78c7d85c1fdb4cb7a257abd13981c3ed, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0dbff6c1562245e685f98086004a8b78, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/04e0fc5478bb4af6b4b49bae4c090361, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/011eb09688c2433f943f854366d6a40c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/135821b0cd544a6096536e0a7a0f8ca2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8dbe8f01f71640d3a7f7bc892eabbad2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/76f24866f576465d97dfb3622b5ff77a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/7bfcb6b5efac465b8398640bef517c9c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/9b81f5f62d1a4ed18aec87e32e545a2f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/3b5b32939f3845388d725c0f85478d00, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2ce41c82f79546e1ab28d27a0a0b9c61, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/e6df701eab9e47e1b4ff224657dd4d36, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2c5539851f384a7c8e7cfeda43ad42f5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f3a0a329353e437b9b89568b3941567f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/802c18089e1149eb878716258be7e888, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/957561514f2e4869bc9ca07ce65b32c0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/408d311c96dd4078a213c993bc2c8713, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/997bbfd16ba141f8b9473f7b5d073d8b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f10a636281334451bba2a35f8178ab5c] to archive 2024-11-11T07:23:38,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:23:38,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0310d27720c54cc58a26c9561dfa8447 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0310d27720c54cc58a26c9561dfa8447 2024-11-11T07:23:38,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8bb99cd4d4024bd6a06e99c478a5531e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8bb99cd4d4024bd6a06e99c478a5531e 2024-11-11T07:23:38,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/da96988635d148ae8f2387b07c08f9c3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/da96988635d148ae8f2387b07c08f9c3 2024-11-11T07:23:38,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/a1060117a66b4bfa983b62fadf4d37f4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/a1060117a66b4bfa983b62fadf4d37f4 2024-11-11T07:23:38,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2a3122ce187a4382bf09e0dd7b0e69a6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2a3122ce187a4382bf09e0dd7b0e69a6 2024-11-11T07:23:38,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c20fa47525b4ab7b6faf7ff4da15d65 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c20fa47525b4ab7b6faf7ff4da15d65 2024-11-11T07:23:38,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/d5caed4041f44f1dad43b1825da54a68 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/d5caed4041f44f1dad43b1825da54a68 2024-11-11T07:23:38,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ee3b6469ade847b0a323bfd786dc7c78 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ee3b6469ade847b0a323bfd786dc7c78 2024-11-11T07:23:38,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/bb4abb4f730943368434331a8201799b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/bb4abb4f730943368434331a8201799b 2024-11-11T07:23:38,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c34832285d5424cbc42aad8144d0b67 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/4c34832285d5424cbc42aad8144d0b67 2024-11-11T07:23:38,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/78c7d85c1fdb4cb7a257abd13981c3ed to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/78c7d85c1fdb4cb7a257abd13981c3ed 2024-11-11T07:23:38,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0dbff6c1562245e685f98086004a8b78 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/0dbff6c1562245e685f98086004a8b78 2024-11-11T07:23:38,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/04e0fc5478bb4af6b4b49bae4c090361 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/04e0fc5478bb4af6b4b49bae4c090361 2024-11-11T07:23:38,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/011eb09688c2433f943f854366d6a40c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/011eb09688c2433f943f854366d6a40c 2024-11-11T07:23:38,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/135821b0cd544a6096536e0a7a0f8ca2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/135821b0cd544a6096536e0a7a0f8ca2 2024-11-11T07:23:38,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8dbe8f01f71640d3a7f7bc892eabbad2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/8dbe8f01f71640d3a7f7bc892eabbad2 2024-11-11T07:23:38,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/76f24866f576465d97dfb3622b5ff77a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/76f24866f576465d97dfb3622b5ff77a 2024-11-11T07:23:38,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/7bfcb6b5efac465b8398640bef517c9c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/7bfcb6b5efac465b8398640bef517c9c 2024-11-11T07:23:38,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/9b81f5f62d1a4ed18aec87e32e545a2f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/9b81f5f62d1a4ed18aec87e32e545a2f 2024-11-11T07:23:38,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/3b5b32939f3845388d725c0f85478d00 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/3b5b32939f3845388d725c0f85478d00 2024-11-11T07:23:38,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2ce41c82f79546e1ab28d27a0a0b9c61 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2ce41c82f79546e1ab28d27a0a0b9c61 2024-11-11T07:23:38,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/e6df701eab9e47e1b4ff224657dd4d36 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/e6df701eab9e47e1b4ff224657dd4d36 2024-11-11T07:23:38,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2c5539851f384a7c8e7cfeda43ad42f5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/2c5539851f384a7c8e7cfeda43ad42f5 2024-11-11T07:23:38,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f3a0a329353e437b9b89568b3941567f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f3a0a329353e437b9b89568b3941567f 2024-11-11T07:23:38,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/802c18089e1149eb878716258be7e888 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/802c18089e1149eb878716258be7e888 2024-11-11T07:23:38,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/957561514f2e4869bc9ca07ce65b32c0 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/957561514f2e4869bc9ca07ce65b32c0 2024-11-11T07:23:38,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/408d311c96dd4078a213c993bc2c8713 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/408d311c96dd4078a213c993bc2c8713 2024-11-11T07:23:38,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/997bbfd16ba141f8b9473f7b5d073d8b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/997bbfd16ba141f8b9473f7b5d073d8b 2024-11-11T07:23:38,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f10a636281334451bba2a35f8178ab5c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/f10a636281334451bba2a35f8178ab5c 2024-11-11T07:23:38,070 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/recovered.edits/460.seqid, newMaxSeqId=460, maxSeqId=1 2024-11-11T07:23:38,070 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. 2024-11-11T07:23:38,070 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for c49bcd3ff91d850f5acc98ee12d10ac3: Waiting for close lock at 1731309815764Running coprocessor pre-close hooks at 1731309815764Disabling compacts and flushes for region at 1731309815764Disabling writes for close at 1731309816729 (+965 ms)Obtaining lock to block concurrent updates at 1731309816729Preparing flush snapshotting stores in c49bcd3ff91d850f5acc98ee12d10ac3 at 1731309816729Finished memstore snapshotting TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731309816729Flushing stores of TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3. at 1731309816730 (+1 ms)Flushing c49bcd3ff91d850f5acc98ee12d10ac3/A: creating writer at 1731309816730Flushing c49bcd3ff91d850f5acc98ee12d10ac3/A: appending metadata at 1731309816732 (+2 ms)Flushing c49bcd3ff91d850f5acc98ee12d10ac3/A: closing flushed file at 1731309816732Flushing c49bcd3ff91d850f5acc98ee12d10ac3/B: creating writer at 1731309817141 (+409 ms)Flushing c49bcd3ff91d850f5acc98ee12d10ac3/B: appending metadata at 1731309817143 (+2 ms)Flushing c49bcd3ff91d850f5acc98ee12d10ac3/B: closing flushed file at 1731309817143Flushing c49bcd3ff91d850f5acc98ee12d10ac3/C: creating writer at 1731309817552 (+409 ms)Flushing c49bcd3ff91d850f5acc98ee12d10ac3/C: appending metadata at 1731309817554 (+2 ms)Flushing c49bcd3ff91d850f5acc98ee12d10ac3/C: closing flushed file at 1731309817554Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4db847b3: reopening flushed file at 1731309817962 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34464664: reopening flushed file at 1731309817966 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30456041: reopening flushed file at 1731309817970 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c49bcd3ff91d850f5acc98ee12d10ac3 in 1245ms, sequenceid=457, compaction requested=true at 1731309817974 (+4 ms)Writing region close event to WAL at 1731309818067 (+93 ms)Running coprocessor post-close hooks at 1731309818070 (+3 ms)Closed at 1731309818070 2024-11-11T07:23:38,072 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:38,073 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=c49bcd3ff91d850f5acc98ee12d10ac3, regionState=CLOSED 2024-11-11T07:23:38,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:23:38,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T07:23:38,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure c49bcd3ff91d850f5acc98ee12d10ac3, server=f334e8b0fbdf,39329,1731309728191 in 2.4640 sec 2024-11-11T07:23:38,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T07:23:38,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49bcd3ff91d850f5acc98ee12d10ac3, UNASSIGN in 2.4670 sec 2024-11-11T07:23:38,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T07:23:38,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4700 sec 2024-11-11T07:23:38,081 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309818081"}]},"ts":"1731309818081"} 2024-11-11T07:23:38,082 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:23:38,082 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:23:38,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4800 sec 2024-11-11T07:23:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:23:39,782 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:39,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:23:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:39,785 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T07:23:39,785 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:39,787 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:39,789 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/recovered.edits] 2024-11-11T07:23:39,792 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4532012c19c941149e2681640051b6cf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/4532012c19c941149e2681640051b6cf 2024-11-11T07:23:39,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/8e040c6d429e4d8daf584586f5adc18f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/8e040c6d429e4d8daf584586f5adc18f 2024-11-11T07:23:39,795 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/a87bc0eb113f4738b865ac8baf25ae46 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/a87bc0eb113f4738b865ac8baf25ae46 2024-11-11T07:23:39,796 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d4022378f1cf4094a087d1be671d2fd4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/A/d4022378f1cf4094a087d1be671d2fd4 2024-11-11T07:23:39,798 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/583d868030fa460983c4cf0676232d44 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/583d868030fa460983c4cf0676232d44 2024-11-11T07:23:39,799 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/69f0e13fbdfb4e96a95f2ab7ea28b5af to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/69f0e13fbdfb4e96a95f2ab7ea28b5af 2024-11-11T07:23:39,801 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/be8dcf58924a45fab362b3c106734a2b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/be8dcf58924a45fab362b3c106734a2b 2024-11-11T07:23:39,802 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/ff0281d229c14734bb719da058d19dec to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/B/ff0281d229c14734bb719da058d19dec 2024-11-11T07:23:39,804 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/939fbed019534fac9cf9e09f05f4b0ad to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/939fbed019534fac9cf9e09f05f4b0ad 2024-11-11T07:23:39,805 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/c234998036924635b72d06b9d3ad96ab to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/c234998036924635b72d06b9d3ad96ab 2024-11-11T07:23:39,806 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ca58dae9b2f94cde8a9d1878769a8b56 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/ca58dae9b2f94cde8a9d1878769a8b56 2024-11-11T07:23:39,807 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/de8593d891ae4c30bbfa377bd9400ef8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/C/de8593d891ae4c30bbfa377bd9400ef8 2024-11-11T07:23:39,809 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/recovered.edits/460.seqid to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3/recovered.edits/460.seqid 2024-11-11T07:23:39,810 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/c49bcd3ff91d850f5acc98ee12d10ac3 2024-11-11T07:23:39,810 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:23:39,812 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:39,816 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:23:39,818 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:23:39,819 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:39,819 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:23:39,820 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731309819819"}]},"ts":"9223372036854775807"} 2024-11-11T07:23:39,822 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:23:39,822 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c49bcd3ff91d850f5acc98ee12d10ac3, NAME => 'TestAcidGuarantees,,1731309792970.c49bcd3ff91d850f5acc98ee12d10ac3.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:23:39,822 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:23:39,822 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731309819822"}]},"ts":"9223372036854775807"} 2024-11-11T07:23:39,824 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:23:39,825 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:39,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-11T07:23:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T07:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:23:39,892 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:39,902 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-21932362_22 at /127.0.0.1:46776 [Waiting for operation #372] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1782211046_22 at /127.0.0.1:46958 [Waiting for operation #241] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=492 (was 374) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7964 (was 8084) 2024-11-11T07:23:39,913 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=492, ProcessCount=12, AvailableMemoryMB=7964 2024-11-11T07:23:39,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:23:39,915 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:23:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:39,917 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:23:39,917 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:39,918 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:23:39,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-11T07:23:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:23:39,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742159_1335 (size=959) 2024-11-11T07:23:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:23:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:23:40,325 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:23:40,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742160_1336 (size=53) 2024-11-11T07:23:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:23:40,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:23:40,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b5411b408cdd3e1e96fba71c599080d2, disabling compactions & flushes 2024-11-11T07:23:40,731 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:40,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:40,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. after waiting 0 ms 2024-11-11T07:23:40,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:40,731 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:40,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b5411b408cdd3e1e96fba71c599080d2: Waiting for close lock at 1731309820731Disabling compacts and flushes for region at 1731309820731Disabling writes for close at 1731309820731Writing region close event to WAL at 1731309820731Closed at 1731309820731 2024-11-11T07:23:40,732 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:23:40,733 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731309820732"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731309820732"}]},"ts":"1731309820732"} 2024-11-11T07:23:40,734 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:23:40,735 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:23:40,735 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309820735"}]},"ts":"1731309820735"} 2024-11-11T07:23:40,736 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:23:40,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, ASSIGN}] 2024-11-11T07:23:40,737 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, ASSIGN 2024-11-11T07:23:40,737 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, ASSIGN; state=OFFLINE, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=false 2024-11-11T07:23:40,888 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:40,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, ASSIGN because future has completed 2024-11-11T07:23:40,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:23:41,044 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:41,045 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:23:41,045 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,045 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:23:41,045 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,045 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,046 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,047 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5411b408cdd3e1e96fba71c599080d2 columnFamilyName A 2024-11-11T07:23:41,047 DEBUG [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:41,048 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(327): Store=b5411b408cdd3e1e96fba71c599080d2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:41,048 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,049 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5411b408cdd3e1e96fba71c599080d2 columnFamilyName B 2024-11-11T07:23:41,049 DEBUG [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:41,049 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(327): Store=b5411b408cdd3e1e96fba71c599080d2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:41,049 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,050 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5411b408cdd3e1e96fba71c599080d2 columnFamilyName C 2024-11-11T07:23:41,050 DEBUG [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:41,050 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(327): Store=b5411b408cdd3e1e96fba71c599080d2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:41,050 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,051 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,051 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:23:41,052 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,052 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,052 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:23:41,053 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,055 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:23:41,055 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened b5411b408cdd3e1e96fba71c599080d2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73121521, jitterRate=0.08959557116031647}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:23:41,055 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:41,056 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for b5411b408cdd3e1e96fba71c599080d2: Running coprocessor pre-open hook at 1731309821045Writing region info on filesystem at 1731309821045Initializing all the Stores at 1731309821046 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309821046Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309821046Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309821046Cleaning up temporary data from old regions at 1731309821052 (+6 ms)Running coprocessor post-open hooks at 1731309821055 (+3 ms)Region opened successfully at 1731309821056 (+1 ms) 2024-11-11T07:23:41,057 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., pid=89, masterSystemTime=1731309821041 2024-11-11T07:23:41,058 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:41,058 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:41,059 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=OPEN, openSeqNum=2, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:41,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:23:41,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T07:23:41,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 in 170 msec 2024-11-11T07:23:41,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T07:23:41,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, ASSIGN in 326 msec 2024-11-11T07:23:41,064 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:23:41,064 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309821064"}]},"ts":"1731309821064"} 2024-11-11T07:23:41,065 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:23:41,065 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:23:41,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-11T07:23:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:23:42,062 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:42,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,064 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c815554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,065 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b97e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,066 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:42,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:42,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:23:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f334e8b0fbdf,39865,1731309727465 2024-11-11T07:23:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@998089c 2024-11-11T07:23:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:23:42,070 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:23:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:23:42,071 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:23:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T07:23:42,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742161_1337 (size=995) 2024-11-11T07:23:42,482 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T07:23:42,482 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T07:23:42,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:23:42,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, REOPEN/MOVE}] 2024-11-11T07:23:42,486 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, REOPEN/MOVE 2024-11-11T07:23:42,487 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:42,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, REOPEN/MOVE because future has completed 2024-11-11T07:23:42,488 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:23:42,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:23:42,640 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,640 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:23:42,640 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing b5411b408cdd3e1e96fba71c599080d2, disabling compactions & flushes 2024-11-11T07:23:42,640 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:42,641 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:42,641 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. after waiting 0 ms 2024-11-11T07:23:42,641 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:42,644 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T07:23:42,644 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:42,644 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for b5411b408cdd3e1e96fba71c599080d2: Waiting for close lock at 1731309822640Running coprocessor pre-close hooks at 1731309822640Disabling compacts and flushes for region at 1731309822640Disabling writes for close at 1731309822641 (+1 ms)Writing region close event to WAL at 1731309822641Running coprocessor post-close hooks at 1731309822644 (+3 ms)Closed at 1731309822644 2024-11-11T07:23:42,644 WARN [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: b5411b408cdd3e1e96fba71c599080d2 to self. 2024-11-11T07:23:42,646 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,646 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=CLOSED 2024-11-11T07:23:42,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:23:42,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T07:23:42,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 in 160 msec 2024-11-11T07:23:42,650 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, REOPEN/MOVE; state=CLOSED, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=true 2024-11-11T07:23:42,801 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:42,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, REOPEN/MOVE because future has completed 2024-11-11T07:23:42,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:23:42,955 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:42,956 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:23:42,956 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,956 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:23:42,956 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,956 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,958 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,958 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5411b408cdd3e1e96fba71c599080d2 columnFamilyName A 2024-11-11T07:23:42,959 DEBUG [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:42,960 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(327): Store=b5411b408cdd3e1e96fba71c599080d2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:42,960 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,960 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5411b408cdd3e1e96fba71c599080d2 columnFamilyName B 2024-11-11T07:23:42,960 DEBUG [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:42,961 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(327): Store=b5411b408cdd3e1e96fba71c599080d2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:42,961 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,962 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5411b408cdd3e1e96fba71c599080d2 columnFamilyName C 2024-11-11T07:23:42,962 DEBUG [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:42,962 INFO [StoreOpener-b5411b408cdd3e1e96fba71c599080d2-1 {}] regionserver.HStore(327): Store=b5411b408cdd3e1e96fba71c599080d2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:23:42,962 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,963 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,964 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,964 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,964 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,965 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:23:42,966 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,967 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened b5411b408cdd3e1e96fba71c599080d2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65175932, jitterRate=-0.028802931308746338}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:23:42,967 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:42,968 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for b5411b408cdd3e1e96fba71c599080d2: Running coprocessor pre-open hook at 1731309822956Writing region info on filesystem at 1731309822956Initializing all the Stores at 1731309822957 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309822957Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309822957Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309822957Cleaning up temporary data from old regions at 1731309822964 (+7 ms)Running coprocessor post-open hooks at 1731309822967 (+3 ms)Region opened successfully at 1731309822968 (+1 ms) 2024-11-11T07:23:42,969 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., pid=94, masterSystemTime=1731309822953 2024-11-11T07:23:42,971 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:42,971 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:42,971 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=OPEN, openSeqNum=5, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:42,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:23:42,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-11T07:23:42,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 in 171 msec 2024-11-11T07:23:42,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T07:23:42,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, REOPEN/MOVE in 488 msec 2024-11-11T07:23:42,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T07:23:42,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-11T07:23:42,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-11T07:23:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T07:23:42,980 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:42,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bda395f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,983 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@975a897, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,985 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9c7ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de184f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e877cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f29aac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,990 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478b9452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@466e2231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b0d4a58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df95477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,995 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@420d7115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,996 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68902cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:42,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:42,998 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538a8f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:42,999 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16714eba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,000 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:43,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:43,000 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d53467a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:43,001 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a0d1d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:43,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:43,003 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c104ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:43,004 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c52a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:23:43,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:23:43,006 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:23:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:23:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:23:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f594170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:23:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:23:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:23:43,007 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:23:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64414b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:23:43,008 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:43,008 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,008 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,008 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,008 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,008 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,009 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,009 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T07:23:43,009 DEBUG [Thread-1542 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,009 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,009 DEBUG [Thread-1542 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,010 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,010 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:23:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,010 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:23:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,012 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:23:43,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:23:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:23:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:23:43,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:43,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:23:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:43,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e4cd55f573554a10a08fcba7729d565c_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309823020/Put/seqid=0 2024-11-11T07:23:43,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742162_1338 (size=14594) 2024-11-11T07:23:43,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:43,059 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e4cd55f573554a10a08fcba7729d565c_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e4cd55f573554a10a08fcba7729d565c_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:43,061 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/a48d037e1eca4220b93ffb616adda526, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:43,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/a48d037e1eca4220b93ffb616adda526 is 175, key is test_row_0/A:col10/1731309823020/Put/seqid=0 2024-11-11T07:23:43,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742163_1339 (size=39549) 2024-11-11T07:23:43,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309883065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309883064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309883073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309883076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309883076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:23:43,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:23:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309883181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309883181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309883191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309883191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309883191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:23:43,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:23:43,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309883391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309883391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309883402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309883402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309883411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/a48d037e1eca4220b93ffb616adda526 2024-11-11T07:23:43,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:23:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/daad491a0aae4780919ea70ec9330274 is 50, key is test_row_0/B:col10/1731309823020/Put/seqid=0 2024-11-11T07:23:43,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742164_1340 (size=12001) 2024-11-11T07:23:43,591 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:23:43,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:23:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:23:43,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309883701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309883701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309883722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309883722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309883722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:43,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:23:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/daad491a0aae4780919ea70ec9330274 2024-11-11T07:23:43,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/8108726c802a4c3c999298bb3c1d7b4b is 50, key is test_row_0/C:col10/1731309823020/Put/seqid=0 2024-11-11T07:23:43,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742165_1341 (size=12001) 2024-11-11T07:23:43,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/8108726c802a4c3c999298bb3c1d7b4b 2024-11-11T07:23:43,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/a48d037e1eca4220b93ffb616adda526 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/a48d037e1eca4220b93ffb616adda526 2024-11-11T07:23:43,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/a48d037e1eca4220b93ffb616adda526, entries=200, sequenceid=16, filesize=38.6 K 2024-11-11T07:23:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/daad491a0aae4780919ea70ec9330274 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/daad491a0aae4780919ea70ec9330274 2024-11-11T07:23:43,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:23:43,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/daad491a0aae4780919ea70ec9330274, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T07:23:43,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:43,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:43,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/8108726c802a4c3c999298bb3c1d7b4b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8108726c802a4c3c999298bb3c1d7b4b 2024-11-11T07:23:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:43,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8108726c802a4c3c999298bb3c1d7b4b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T07:23:43,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b5411b408cdd3e1e96fba71c599080d2 in 921ms, sequenceid=16, compaction requested=false 2024-11-11T07:23:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:44,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:23:44,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:44,095 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:23:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119826b6a5a34a42eeb8488790a05fb78c_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309823063/Put/seqid=0 2024-11-11T07:23:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742166_1342 (size=12154) 2024-11-11T07:23:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:23:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:44,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309884241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309884242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309884242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309884242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309884246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309884352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309884362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:44,512 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119826b6a5a34a42eeb8488790a05fb78c_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119826b6a5a34a42eeb8488790a05fb78c_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/4257ceaa9d174c5a829634d2a79af2cd, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/4257ceaa9d174c5a829634d2a79af2cd is 175, key is test_row_0/A:col10/1731309823063/Put/seqid=0 2024-11-11T07:23:44,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742167_1343 (size=30955) 2024-11-11T07:23:44,523 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/4257ceaa9d174c5a829634d2a79af2cd 2024-11-11T07:23:44,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/4cc79c4c1000423aaac1c82183c841a4 is 50, key is test_row_0/B:col10/1731309823063/Put/seqid=0 2024-11-11T07:23:44,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742168_1344 (size=12001) 2024-11-11T07:23:44,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309884562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309884572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309884871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309884881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:44,942 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/4cc79c4c1000423aaac1c82183c841a4 2024-11-11T07:23:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/337f2777d0d14b0cb9fe5205311086f9 is 50, key is test_row_0/C:col10/1731309823063/Put/seqid=0 2024-11-11T07:23:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742169_1345 (size=12001) 2024-11-11T07:23:44,954 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/337f2777d0d14b0cb9fe5205311086f9 2024-11-11T07:23:44,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/4257ceaa9d174c5a829634d2a79af2cd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/4257ceaa9d174c5a829634d2a79af2cd 2024-11-11T07:23:44,962 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/4257ceaa9d174c5a829634d2a79af2cd, entries=150, sequenceid=40, filesize=30.2 K 2024-11-11T07:23:44,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/4cc79c4c1000423aaac1c82183c841a4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/4cc79c4c1000423aaac1c82183c841a4 2024-11-11T07:23:44,967 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/4cc79c4c1000423aaac1c82183c841a4, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T07:23:44,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/337f2777d0d14b0cb9fe5205311086f9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/337f2777d0d14b0cb9fe5205311086f9 2024-11-11T07:23:44,973 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/337f2777d0d14b0cb9fe5205311086f9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T07:23:44,974 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b5411b408cdd3e1e96fba71c599080d2 in 879ms, sequenceid=40, compaction requested=false 2024-11-11T07:23:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T07:23:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T07:23:44,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T07:23:44,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9640 sec 2024-11-11T07:23:44,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9700 sec 2024-11-11T07:23:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:23:45,162 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:45,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T07:23:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:23:45,164 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:45,164 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:45,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:45,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:45,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:23:45,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e31e41373c5543039484c877812f6fce_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309825253/Put/seqid=0 2024-11-11T07:23:45,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742170_1346 (size=14594) 2024-11-11T07:23:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:23:45,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309885315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309885319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309885320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309885382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309885401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309885431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309885431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309885442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:23:45,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309885651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309885651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309885651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,668 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:45,671 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e31e41373c5543039484c877812f6fce_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e31e41373c5543039484c877812f6fce_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:45,671 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/516fff8944ca4de3a1872b7c2f420144, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:45,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/516fff8944ca4de3a1872b7c2f420144 is 175, key is test_row_0/A:col10/1731309825253/Put/seqid=0 2024-11-11T07:23:45,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742171_1347 (size=39549) 2024-11-11T07:23:45,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:45,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:45,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:23:45,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:45,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:45,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:45,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:45,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309885971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309885971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309885971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:46,076 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/516fff8944ca4de3a1872b7c2f420144 2024-11-11T07:23:46,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/25fb20afa3244808952ddca337fbe5e8 is 50, key is test_row_0/B:col10/1731309825253/Put/seqid=0 2024-11-11T07:23:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742172_1348 (size=12001) 2024-11-11T07:23:46,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/25fb20afa3244808952ddca337fbe5e8 2024-11-11T07:23:46,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/8e895cb597b94eb7b08978e3383f6ee8 is 50, key is test_row_0/C:col10/1731309825253/Put/seqid=0 2024-11-11T07:23:46,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742173_1349 (size=12001) 2024-11-11T07:23:46,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:46,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:46,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:23:46,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:46,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309886401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:46,411 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309886422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:46,429 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:46,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309886481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:46,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309886482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:46,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309886482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:46,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/8e895cb597b94eb7b08978e3383f6ee8 2024-11-11T07:23:46,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/516fff8944ca4de3a1872b7c2f420144 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/516fff8944ca4de3a1872b7c2f420144 2024-11-11T07:23:46,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/516fff8944ca4de3a1872b7c2f420144, entries=200, sequenceid=53, filesize=38.6 K 2024-11-11T07:23:46,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/25fb20afa3244808952ddca337fbe5e8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/25fb20afa3244808952ddca337fbe5e8 2024-11-11T07:23:46,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/25fb20afa3244808952ddca337fbe5e8, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T07:23:46,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/8e895cb597b94eb7b08978e3383f6ee8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8e895cb597b94eb7b08978e3383f6ee8 2024-11-11T07:23:46,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8e895cb597b94eb7b08978e3383f6ee8, entries=150, sequenceid=53, filesize=11.7 K 2024-11-11T07:23:46,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b5411b408cdd3e1e96fba71c599080d2 in 1265ms, sequenceid=53, compaction requested=true 2024-11-11T07:23:46,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:46,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:46,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:46,520 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:46,521 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:46,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:46,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:46,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:46,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:46,521 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:46,521 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:46,522 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/B is initiating minor compaction (all files) 2024-11-11T07:23:46,522 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/A is initiating minor compaction (all files) 2024-11-11T07:23:46,522 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/A in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,522 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/B in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,522 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/a48d037e1eca4220b93ffb616adda526, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/4257ceaa9d174c5a829634d2a79af2cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/516fff8944ca4de3a1872b7c2f420144] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=107.5 K 2024-11-11T07:23:46,522 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/daad491a0aae4780919ea70ec9330274, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/4cc79c4c1000423aaac1c82183c841a4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/25fb20afa3244808952ddca337fbe5e8] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=35.2 K 2024-11-11T07:23:46,522 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,522 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/a48d037e1eca4220b93ffb616adda526, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/4257ceaa9d174c5a829634d2a79af2cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/516fff8944ca4de3a1872b7c2f420144] 2024-11-11T07:23:46,522 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48d037e1eca4220b93ffb616adda526, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731309823020 2024-11-11T07:23:46,522 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting daad491a0aae4780919ea70ec9330274, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731309823020 2024-11-11T07:23:46,523 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc79c4c1000423aaac1c82183c841a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731309823063 2024-11-11T07:23:46,523 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4257ceaa9d174c5a829634d2a79af2cd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731309823063 2024-11-11T07:23:46,523 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 25fb20afa3244808952ddca337fbe5e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309824219 2024-11-11T07:23:46,523 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516fff8944ca4de3a1872b7c2f420144, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309824219 2024-11-11T07:23:46,531 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:46,533 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111185db666fdc6a428b8e68ad7a8b42d8bb_b5411b408cdd3e1e96fba71c599080d2 store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:46,534 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:46,534 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/857a063be7d94fb3bab81d5f53c3948e is 50, key is test_row_0/B:col10/1731309825253/Put/seqid=0 2024-11-11T07:23:46,536 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111185db666fdc6a428b8e68ad7a8b42d8bb_b5411b408cdd3e1e96fba71c599080d2, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:46,536 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111185db666fdc6a428b8e68ad7a8b42d8bb_b5411b408cdd3e1e96fba71c599080d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:46,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:23:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,541 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:23:46,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742174_1350 (size=12104) 2024-11-11T07:23:46,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742175_1351 (size=4469) 2024-11-11T07:23:46,555 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#A#compaction#302 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:46,556 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/001c78b3d658479f8beeca4ffb2591bf is 175, key is test_row_0/A:col10/1731309825253/Put/seqid=0 2024-11-11T07:23:46,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118869e721d9f541d9b21d52cccec9152d_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309825318/Put/seqid=0 2024-11-11T07:23:46,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742176_1352 (size=31058) 2024-11-11T07:23:46,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742177_1353 (size=12154) 2024-11-11T07:23:46,955 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/857a063be7d94fb3bab81d5f53c3948e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/857a063be7d94fb3bab81d5f53c3948e 2024-11-11T07:23:46,959 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/B of b5411b408cdd3e1e96fba71c599080d2 into 857a063be7d94fb3bab81d5f53c3948e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:46,959 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:46,959 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/B, priority=13, startTime=1731309826521; duration=0sec 2024-11-11T07:23:46,959 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:46,959 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:B 2024-11-11T07:23:46,959 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:46,960 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:46,960 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/C is initiating minor compaction (all files) 2024-11-11T07:23:46,960 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/C in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:46,960 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8108726c802a4c3c999298bb3c1d7b4b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/337f2777d0d14b0cb9fe5205311086f9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8e895cb597b94eb7b08978e3383f6ee8] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=35.2 K 2024-11-11T07:23:46,960 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8108726c802a4c3c999298bb3c1d7b4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731309823020 2024-11-11T07:23:46,961 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 337f2777d0d14b0cb9fe5205311086f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731309823063 2024-11-11T07:23:46,961 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e895cb597b94eb7b08978e3383f6ee8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309824219 2024-11-11T07:23:46,968 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:46,968 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/924d6f0ab1cf4c0697829ffd5c0b0f62 is 50, key is test_row_0/C:col10/1731309825253/Put/seqid=0 2024-11-11T07:23:46,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742178_1354 (size=12104) 2024-11-11T07:23:46,977 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/924d6f0ab1cf4c0697829ffd5c0b0f62 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/924d6f0ab1cf4c0697829ffd5c0b0f62 2024-11-11T07:23:46,983 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/C of b5411b408cdd3e1e96fba71c599080d2 into 924d6f0ab1cf4c0697829ffd5c0b0f62(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:46,983 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:46,983 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/C, priority=13, startTime=1731309826521; duration=0sec 2024-11-11T07:23:46,983 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:46,983 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:C 2024-11-11T07:23:46,987 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/001c78b3d658479f8beeca4ffb2591bf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/001c78b3d658479f8beeca4ffb2591bf 2024-11-11T07:23:46,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:46,992 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/A of b5411b408cdd3e1e96fba71c599080d2 into 001c78b3d658479f8beeca4ffb2591bf(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:46,992 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:46,992 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/A, priority=13, startTime=1731309826520; duration=0sec 2024-11-11T07:23:46,992 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:46,992 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:A 2024-11-11T07:23:46,994 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118869e721d9f541d9b21d52cccec9152d_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118869e721d9f541d9b21d52cccec9152d_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:46,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7bcfcc3e0b24439bbe044f2195655f5d, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:46,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7bcfcc3e0b24439bbe044f2195655f5d is 175, key is test_row_0/A:col10/1731309825318/Put/seqid=0 2024-11-11T07:23:46,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742179_1355 (size=30955) 2024-11-11T07:23:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:23:47,400 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7bcfcc3e0b24439bbe044f2195655f5d 2024-11-11T07:23:47,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/39ea2550643e4ae8bbf0ea1ee332e295 is 50, key is test_row_0/B:col10/1731309825318/Put/seqid=0 2024-11-11T07:23:47,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742180_1356 (size=12001) 2024-11-11T07:23:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:47,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:47,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309887528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309887528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309887530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309887642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309887642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309887652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,812 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/39ea2550643e4ae8bbf0ea1ee332e295 2024-11-11T07:23:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/aecd15498715459888320a2a0ad2613a is 50, key is test_row_0/C:col10/1731309825318/Put/seqid=0 2024-11-11T07:23:47,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742181_1357 (size=12001) 2024-11-11T07:23:47,824 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/aecd15498715459888320a2a0ad2613a 2024-11-11T07:23:47,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7bcfcc3e0b24439bbe044f2195655f5d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7bcfcc3e0b24439bbe044f2195655f5d 2024-11-11T07:23:47,831 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7bcfcc3e0b24439bbe044f2195655f5d, entries=150, sequenceid=76, filesize=30.2 K 2024-11-11T07:23:47,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/39ea2550643e4ae8bbf0ea1ee332e295 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/39ea2550643e4ae8bbf0ea1ee332e295 2024-11-11T07:23:47,835 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/39ea2550643e4ae8bbf0ea1ee332e295, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T07:23:47,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/aecd15498715459888320a2a0ad2613a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/aecd15498715459888320a2a0ad2613a 2024-11-11T07:23:47,840 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/aecd15498715459888320a2a0ad2613a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T07:23:47,841 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b5411b408cdd3e1e96fba71c599080d2 in 1300ms, sequenceid=76, compaction requested=false 2024-11-11T07:23:47,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:47,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:47,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T07:23:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T07:23:47,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T07:23:47,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6790 sec 2024-11-11T07:23:47,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.6830 sec 2024-11-11T07:23:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:47,853 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:47,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:23:47,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ae15cc245cdc4b0e9895888ad3d077d3_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309827526/Put/seqid=0 2024-11-11T07:23:47,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742182_1358 (size=17034) 2024-11-11T07:23:47,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:47,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ae15cc245cdc4b0e9895888ad3d077d3_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae15cc245cdc4b0e9895888ad3d077d3_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:47,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7de5357a61d34d9894d21b325de56f71, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:47,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7de5357a61d34d9894d21b325de56f71 is 175, key is test_row_0/A:col10/1731309827526/Put/seqid=0 2024-11-11T07:23:47,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742183_1359 (size=48139) 2024-11-11T07:23:47,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7de5357a61d34d9894d21b325de56f71 2024-11-11T07:23:47,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/f3cc01a5ee054c919b1104710e1cfa48 is 50, key is test_row_0/B:col10/1731309827526/Put/seqid=0 2024-11-11T07:23:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742184_1360 (size=12001) 2024-11-11T07:23:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309887911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309887918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309887920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309888031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309888031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309888031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309888251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309888251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309888251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/f3cc01a5ee054c919b1104710e1cfa48 2024-11-11T07:23:48,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e90ed97e5caa4aa7ac85f8267d35b477 is 50, key is test_row_0/C:col10/1731309827526/Put/seqid=0 2024-11-11T07:23:48,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742185_1361 (size=12001) 2024-11-11T07:23:48,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309888432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,443 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309888442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,450 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309888562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309888562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309888562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:48,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e90ed97e5caa4aa7ac85f8267d35b477 2024-11-11T07:23:48,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7de5357a61d34d9894d21b325de56f71 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7de5357a61d34d9894d21b325de56f71 2024-11-11T07:23:48,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7de5357a61d34d9894d21b325de56f71, entries=250, sequenceid=94, filesize=47.0 K 2024-11-11T07:23:48,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/f3cc01a5ee054c919b1104710e1cfa48 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f3cc01a5ee054c919b1104710e1cfa48 2024-11-11T07:23:48,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f3cc01a5ee054c919b1104710e1cfa48, entries=150, sequenceid=94, filesize=11.7 K 2024-11-11T07:23:48,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e90ed97e5caa4aa7ac85f8267d35b477 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e90ed97e5caa4aa7ac85f8267d35b477 2024-11-11T07:23:48,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e90ed97e5caa4aa7ac85f8267d35b477, entries=150, sequenceid=94, filesize=11.7 K 2024-11-11T07:23:48,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b5411b408cdd3e1e96fba71c599080d2 in 865ms, sequenceid=94, compaction requested=true 2024-11-11T07:23:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:48,719 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:48,719 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:48,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:48,720 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:48,720 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:48,720 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/B is initiating minor compaction (all files) 2024-11-11T07:23:48,720 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/A is initiating minor compaction (all files) 2024-11-11T07:23:48,720 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/B in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:48,720 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/A in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:48,720 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/857a063be7d94fb3bab81d5f53c3948e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/39ea2550643e4ae8bbf0ea1ee332e295, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f3cc01a5ee054c919b1104710e1cfa48] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=35.3 K 2024-11-11T07:23:48,720 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/001c78b3d658479f8beeca4ffb2591bf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7bcfcc3e0b24439bbe044f2195655f5d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7de5357a61d34d9894d21b325de56f71] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=107.6 K 2024-11-11T07:23:48,720 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:48,720 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/001c78b3d658479f8beeca4ffb2591bf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7bcfcc3e0b24439bbe044f2195655f5d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7de5357a61d34d9894d21b325de56f71] 2024-11-11T07:23:48,720 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 857a063be7d94fb3bab81d5f53c3948e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309824219 2024-11-11T07:23:48,721 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001c78b3d658479f8beeca4ffb2591bf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309824219 2024-11-11T07:23:48,721 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ea2550643e4ae8bbf0ea1ee332e295, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731309825304 2024-11-11T07:23:48,721 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bcfcc3e0b24439bbe044f2195655f5d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731309825304 2024-11-11T07:23:48,721 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f3cc01a5ee054c919b1104710e1cfa48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731309827526 2024-11-11T07:23:48,722 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de5357a61d34d9894d21b325de56f71, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731309827526 2024-11-11T07:23:48,728 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#B#compaction#311 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:48,728 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/7fde3001ac4946d19d9ae2942f3df966 is 50, key is test_row_0/B:col10/1731309827526/Put/seqid=0 2024-11-11T07:23:48,729 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:48,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742186_1362 (size=12207) 2024-11-11T07:23:48,733 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115775f2a784314e8cbb8e0b1f8d44a59b_b5411b408cdd3e1e96fba71c599080d2 store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:48,735 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115775f2a784314e8cbb8e0b1f8d44a59b_b5411b408cdd3e1e96fba71c599080d2, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:48,735 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115775f2a784314e8cbb8e0b1f8d44a59b_b5411b408cdd3e1e96fba71c599080d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:48,738 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/7fde3001ac4946d19d9ae2942f3df966 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/7fde3001ac4946d19d9ae2942f3df966 2024-11-11T07:23:48,745 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/B of b5411b408cdd3e1e96fba71c599080d2 into 7fde3001ac4946d19d9ae2942f3df966(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:48,745 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:48,745 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/B, priority=13, startTime=1731309828719; duration=0sec 2024-11-11T07:23:48,745 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:48,745 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:B 2024-11-11T07:23:48,745 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:48,748 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:48,748 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/C is initiating minor compaction (all files) 2024-11-11T07:23:48,748 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/C in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:48,748 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/924d6f0ab1cf4c0697829ffd5c0b0f62, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/aecd15498715459888320a2a0ad2613a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e90ed97e5caa4aa7ac85f8267d35b477] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=35.3 K 2024-11-11T07:23:48,748 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 924d6f0ab1cf4c0697829ffd5c0b0f62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731309824219 2024-11-11T07:23:48,749 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting aecd15498715459888320a2a0ad2613a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731309825304 2024-11-11T07:23:48,749 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e90ed97e5caa4aa7ac85f8267d35b477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731309827526 2024-11-11T07:23:48,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742187_1363 (size=4469) 2024-11-11T07:23:48,764 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#C#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:48,764 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/a2585214e8564baea6a25739d9464793 is 50, key is test_row_0/C:col10/1731309827526/Put/seqid=0 2024-11-11T07:23:48,766 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#A#compaction#312 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:48,767 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/d8392e9eef884b1195c0907906939c25 is 175, key is test_row_0/A:col10/1731309827526/Put/seqid=0 2024-11-11T07:23:48,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742188_1364 (size=12207) 2024-11-11T07:23:48,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742189_1365 (size=31161) 2024-11-11T07:23:48,787 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/d8392e9eef884b1195c0907906939c25 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/d8392e9eef884b1195c0907906939c25 2024-11-11T07:23:48,794 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/A of b5411b408cdd3e1e96fba71c599080d2 into d8392e9eef884b1195c0907906939c25(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:48,794 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:48,794 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/A, priority=13, startTime=1731309828719; duration=0sec 2024-11-11T07:23:48,794 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:48,794 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:A 2024-11-11T07:23:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:49,075 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:49,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T07:23:49,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b3cc9fc6e18493fb54fadc5246b5286_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309829074/Put/seqid=0 2024-11-11T07:23:49,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742190_1366 (size=14594) 2024-11-11T07:23:49,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:49,092 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112b3cc9fc6e18493fb54fadc5246b5286_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b3cc9fc6e18493fb54fadc5246b5286_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:49,093 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/94b3c1ea444942bab47abbf35a9b90e5, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:49,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/94b3c1ea444942bab47abbf35a9b90e5 is 175, key is test_row_0/A:col10/1731309829074/Put/seqid=0 2024-11-11T07:23:49,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742191_1367 (size=39549) 2024-11-11T07:23:49,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309889103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309889104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309889112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,177 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/a2585214e8564baea6a25739d9464793 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/a2585214e8564baea6a25739d9464793 2024-11-11T07:23:49,181 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/C of b5411b408cdd3e1e96fba71c599080d2 into a2585214e8564baea6a25739d9464793(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:49,181 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:49,181 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/C, priority=13, startTime=1731309828719; duration=0sec 2024-11-11T07:23:49,182 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:49,182 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:C 2024-11-11T07:23:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309889222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309889222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309889222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:23:49,332 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:49,333 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T07:23:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:23:49,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:49,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:49,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309889431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309889431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:23:49,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309889442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:23:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:49,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/94b3c1ea444942bab47abbf35a9b90e5 2024-11-11T07:23:49,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/33fb4d85117a4a53bb254548ec971563 is 50, key is test_row_0/B:col10/1731309829074/Put/seqid=0 2024-11-11T07:23:49,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742192_1368 (size=12001) 2024-11-11T07:23:49,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:23:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:49,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:23:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309889741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309889741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309889751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:49,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:23:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:49,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:49,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/33fb4d85117a4a53bb254548ec971563 2024-11-11T07:23:49,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/c21c50b29a2b4b98a4aca7ceb7dcc75a is 50, key is test_row_0/C:col10/1731309829074/Put/seqid=0 2024-11-11T07:23:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742193_1369 (size=12001) 2024-11-11T07:23:49,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/c21c50b29a2b4b98a4aca7ceb7dcc75a 2024-11-11T07:23:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/94b3c1ea444942bab47abbf35a9b90e5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/94b3c1ea444942bab47abbf35a9b90e5 2024-11-11T07:23:49,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/94b3c1ea444942bab47abbf35a9b90e5, entries=200, sequenceid=118, filesize=38.6 K 2024-11-11T07:23:49,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/33fb4d85117a4a53bb254548ec971563 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/33fb4d85117a4a53bb254548ec971563 2024-11-11T07:23:49,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/33fb4d85117a4a53bb254548ec971563, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T07:23:49,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/c21c50b29a2b4b98a4aca7ceb7dcc75a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c21c50b29a2b4b98a4aca7ceb7dcc75a 2024-11-11T07:23:49,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c21c50b29a2b4b98a4aca7ceb7dcc75a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T07:23:49,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for b5411b408cdd3e1e96fba71c599080d2 in 863ms, sequenceid=118, compaction requested=false 2024-11-11T07:23:49,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:49,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:23:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:49,945 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:23:49,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110eec6270d2904ec197181099eec13c1a_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309829103/Put/seqid=0 2024-11-11T07:23:49,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742194_1370 (size=12254) 2024-11-11T07:23:49,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:49,960 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110eec6270d2904ec197181099eec13c1a_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110eec6270d2904ec197181099eec13c1a_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:23:49,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/25337c77546e450e834af2ead537bee0, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:49,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/25337c77546e450e834af2ead537bee0 is 175, key is test_row_0/A:col10/1731309829103/Put/seqid=0 2024-11-11T07:23:49,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742195_1371 (size=31055) 2024-11-11T07:23:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:50,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309890341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309890343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309890352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,367 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/25337c77546e450e834af2ead537bee0 2024-11-11T07:23:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/cacfcbcf84bd40ecbe434f0c4f167795 is 50, key is test_row_0/B:col10/1731309829103/Put/seqid=0 2024-11-11T07:23:50,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742196_1372 (size=12101) 2024-11-11T07:23:50,381 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/cacfcbcf84bd40ecbe434f0c4f167795 2024-11-11T07:23:50,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e62717e91ef64d14af61165bc9bb0591 is 50, key is test_row_0/C:col10/1731309829103/Put/seqid=0 2024-11-11T07:23:50,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742197_1373 (size=12101) 2024-11-11T07:23:50,398 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e62717e91ef64d14af61165bc9bb0591 2024-11-11T07:23:50,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/25337c77546e450e834af2ead537bee0 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/25337c77546e450e834af2ead537bee0 2024-11-11T07:23:50,408 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/25337c77546e450e834af2ead537bee0, entries=150, sequenceid=133, filesize=30.3 K 2024-11-11T07:23:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/cacfcbcf84bd40ecbe434f0c4f167795 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/cacfcbcf84bd40ecbe434f0c4f167795 2024-11-11T07:23:50,416 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/cacfcbcf84bd40ecbe434f0c4f167795, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T07:23:50,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e62717e91ef64d14af61165bc9bb0591 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e62717e91ef64d14af61165bc9bb0591 2024-11-11T07:23:50,421 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e62717e91ef64d14af61165bc9bb0591, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T07:23:50,422 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b5411b408cdd3e1e96fba71c599080d2 in 477ms, sequenceid=133, compaction requested=true 2024-11-11T07:23:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T07:23:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T07:23:50,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T07:23:50,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0890 sec 2024-11-11T07:23:50,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.0930 sec 2024-11-11T07:23:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:50,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:50,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:23:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:23:50,472 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:50,472 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T07:23:50,474 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:23:50,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114980ae1b791d4950af187ed19a5c1c43_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309830351/Put/seqid=0 2024-11-11T07:23:50,476 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:50,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742198_1374 (size=14794) 2024-11-11T07:23:50,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:50,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114980ae1b791d4950af187ed19a5c1c43_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114980ae1b791d4950af187ed19a5c1c43_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:50,491 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/015ae56fb0874a5e9e20f5633a688769, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:50,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/015ae56fb0874a5e9e20f5633a688769 is 175, key is test_row_0/A:col10/1731309830351/Put/seqid=0 2024-11-11T07:23:50,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742199_1375 (size=39749) 2024-11-11T07:23:50,508 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/015ae56fb0874a5e9e20f5633a688769 2024-11-11T07:23:50,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/75cad00877754477b9ec10b91189348a is 50, key is test_row_0/B:col10/1731309830351/Put/seqid=0 2024-11-11T07:23:50,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309890507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309890511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309890522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742200_1376 (size=12151) 2024-11-11T07:23:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:23:50,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T07:23:50,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:50,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:50,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:50,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309890631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309890631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309890641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T07:23:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:50,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:23:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309890851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309890851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309890861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:50,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/75cad00877754477b9ec10b91189348a 2024-11-11T07:23:50,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T07:23:50,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:50,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:50,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:50,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:50,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e58c851e12f146309ecb7e068722b0fe is 50, key is test_row_0/C:col10/1731309830351/Put/seqid=0 2024-11-11T07:23:50,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742201_1377 (size=12151) 2024-11-11T07:23:51,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T07:23:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:23:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309891171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309891171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309891182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T07:23:51,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:51,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:51,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:51,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e58c851e12f146309ecb7e068722b0fe 2024-11-11T07:23:51,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/015ae56fb0874a5e9e20f5633a688769 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/015ae56fb0874a5e9e20f5633a688769 2024-11-11T07:23:51,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/015ae56fb0874a5e9e20f5633a688769, entries=200, sequenceid=156, filesize=38.8 K 2024-11-11T07:23:51,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/75cad00877754477b9ec10b91189348a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/75cad00877754477b9ec10b91189348a 2024-11-11T07:23:51,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/75cad00877754477b9ec10b91189348a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:23:51,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e58c851e12f146309ecb7e068722b0fe as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e58c851e12f146309ecb7e068722b0fe 2024-11-11T07:23:51,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e58c851e12f146309ecb7e068722b0fe, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:23:51,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b5411b408cdd3e1e96fba71c599080d2 in 913ms, sequenceid=156, compaction requested=true 2024-11-11T07:23:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:51,377 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:51,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:51,377 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:51,378 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:51,378 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:51,379 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/A is initiating minor compaction (all files) 2024-11-11T07:23:51,379 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/B is initiating minor compaction (all files) 2024-11-11T07:23:51,379 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/B in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,379 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/A in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,379 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/7fde3001ac4946d19d9ae2942f3df966, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/33fb4d85117a4a53bb254548ec971563, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/cacfcbcf84bd40ecbe434f0c4f167795, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/75cad00877754477b9ec10b91189348a] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=47.3 K 2024-11-11T07:23:51,379 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/d8392e9eef884b1195c0907906939c25, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/94b3c1ea444942bab47abbf35a9b90e5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/25337c77546e450e834af2ead537bee0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/015ae56fb0874a5e9e20f5633a688769] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=138.2 K 2024-11-11T07:23:51,379 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,379 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/d8392e9eef884b1195c0907906939c25, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/94b3c1ea444942bab47abbf35a9b90e5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/25337c77546e450e834af2ead537bee0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/015ae56fb0874a5e9e20f5633a688769] 2024-11-11T07:23:51,379 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fde3001ac4946d19d9ae2942f3df966, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731309827526 2024-11-11T07:23:51,379 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8392e9eef884b1195c0907906939c25, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731309827526 2024-11-11T07:23:51,380 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 33fb4d85117a4a53bb254548ec971563, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731309827904 2024-11-11T07:23:51,380 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b3c1ea444942bab47abbf35a9b90e5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731309827904 2024-11-11T07:23:51,382 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cacfcbcf84bd40ecbe434f0c4f167795, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731309829089 2024-11-11T07:23:51,382 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25337c77546e450e834af2ead537bee0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731309829089 2024-11-11T07:23:51,382 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 75cad00877754477b9ec10b91189348a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309830328 2024-11-11T07:23:51,382 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 015ae56fb0874a5e9e20f5633a688769, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309830328 2024-11-11T07:23:51,390 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:51,391 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/59bb0e43e8e44cd187affe8660f26d39 is 50, key is test_row_0/B:col10/1731309830351/Put/seqid=0 2024-11-11T07:23:51,397 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:51,402 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111106292be9906c4040a7729eb6d29c8276_b5411b408cdd3e1e96fba71c599080d2 store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:51,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742202_1378 (size=12493) 2024-11-11T07:23:51,404 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111106292be9906c4040a7729eb6d29c8276_b5411b408cdd3e1e96fba71c599080d2, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:51,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T07:23:51,404 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111106292be9906c4040a7729eb6d29c8276_b5411b408cdd3e1e96fba71c599080d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:51,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,405 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:23:51,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742203_1379 (size=4469) 2024-11-11T07:23:51,418 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#A#compaction#324 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:51,418 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/888e0060cd7c4661b7ea31f8492617cc is 175, key is test_row_0/A:col10/1731309830351/Put/seqid=0 2024-11-11T07:23:51,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ca76703a0e84d80a3027e78ff32332f_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309830507/Put/seqid=0 2024-11-11T07:23:51,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742204_1380 (size=31447) 2024-11-11T07:23:51,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742205_1381 (size=12304) 2024-11-11T07:23:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:51,455 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ca76703a0e84d80a3027e78ff32332f_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ca76703a0e84d80a3027e78ff32332f_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8c9c426a7e52422daa3f88e8497cb211, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:51,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8c9c426a7e52422daa3f88e8497cb211 is 175, key is test_row_0/A:col10/1731309830507/Put/seqid=0 2024-11-11T07:23:51,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742206_1382 (size=31105) 2024-11-11T07:23:51,464 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8c9c426a7e52422daa3f88e8497cb211 2024-11-11T07:23:51,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/0c6466af750744b79b99c2f0b9f660e3 is 50, key is test_row_0/B:col10/1731309830507/Put/seqid=0 2024-11-11T07:23:51,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742207_1383 (size=12151) 2024-11-11T07:23:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:23:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:51,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:51,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309891767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309891768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309891770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,808 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/59bb0e43e8e44cd187affe8660f26d39 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/59bb0e43e8e44cd187affe8660f26d39 2024-11-11T07:23:51,814 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/B of b5411b408cdd3e1e96fba71c599080d2 into 59bb0e43e8e44cd187affe8660f26d39(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:51,814 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:51,814 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/B, priority=12, startTime=1731309831377; duration=0sec 2024-11-11T07:23:51,814 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:51,814 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:B 2024-11-11T07:23:51,814 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:23:51,817 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:23:51,817 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/C is initiating minor compaction (all files) 2024-11-11T07:23:51,817 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/C in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:51,817 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/a2585214e8564baea6a25739d9464793, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c21c50b29a2b4b98a4aca7ceb7dcc75a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e62717e91ef64d14af61165bc9bb0591, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e58c851e12f146309ecb7e068722b0fe] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=47.3 K 2024-11-11T07:23:51,818 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a2585214e8564baea6a25739d9464793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731309827526 2024-11-11T07:23:51,818 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c21c50b29a2b4b98a4aca7ceb7dcc75a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731309827904 2024-11-11T07:23:51,818 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e62717e91ef64d14af61165bc9bb0591, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731309829089 2024-11-11T07:23:51,821 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e58c851e12f146309ecb7e068722b0fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309830328 2024-11-11T07:23:51,829 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:51,829 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/d5ea08dda0a94ae1ad8534ed3409c318 is 50, key is test_row_0/C:col10/1731309830351/Put/seqid=0 2024-11-11T07:23:51,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742208_1384 (size=12493) 2024-11-11T07:23:51,850 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/d5ea08dda0a94ae1ad8534ed3409c318 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d5ea08dda0a94ae1ad8534ed3409c318 2024-11-11T07:23:51,850 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/888e0060cd7c4661b7ea31f8492617cc as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/888e0060cd7c4661b7ea31f8492617cc 2024-11-11T07:23:51,855 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/A of b5411b408cdd3e1e96fba71c599080d2 into 888e0060cd7c4661b7ea31f8492617cc(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:51,855 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/C of b5411b408cdd3e1e96fba71c599080d2 into d5ea08dda0a94ae1ad8534ed3409c318(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:51,855 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:51,855 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:51,855 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/A, priority=12, startTime=1731309831377; duration=0sec 2024-11-11T07:23:51,855 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/C, priority=12, startTime=1731309831377; duration=0sec 2024-11-11T07:23:51,855 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:51,855 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:A 2024-11-11T07:23:51,856 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:51,856 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:C 2024-11-11T07:23:51,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309891872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,883 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/0c6466af750744b79b99c2f0b9f660e3 2024-11-11T07:23:51,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/f83ba60b95ba48a697ddeafd8b677df0 is 50, key is test_row_0/C:col10/1731309830507/Put/seqid=0 2024-11-11T07:23:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309891891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309891891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:51,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742209_1385 (size=12151) 2024-11-11T07:23:52,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309892091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309892101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309892101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,307 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/f83ba60b95ba48a697ddeafd8b677df0 2024-11-11T07:23:52,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8c9c426a7e52422daa3f88e8497cb211 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8c9c426a7e52422daa3f88e8497cb211 2024-11-11T07:23:52,317 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8c9c426a7e52422daa3f88e8497cb211, entries=150, sequenceid=169, filesize=30.4 K 2024-11-11T07:23:52,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/0c6466af750744b79b99c2f0b9f660e3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/0c6466af750744b79b99c2f0b9f660e3 2024-11-11T07:23:52,321 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/0c6466af750744b79b99c2f0b9f660e3, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T07:23:52,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/f83ba60b95ba48a697ddeafd8b677df0 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f83ba60b95ba48a697ddeafd8b677df0 2024-11-11T07:23:52,325 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f83ba60b95ba48a697ddeafd8b677df0, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T07:23:52,326 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b5411b408cdd3e1e96fba71c599080d2 in 921ms, sequenceid=169, compaction requested=false 2024-11-11T07:23:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:52,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:52,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T07:23:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T07:23:52,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T07:23:52,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-11-11T07:23:52,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8580 sec 2024-11-11T07:23:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:52,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:52,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:23:52,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111163234171d7f841748587cebfd28de11f_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309832406/Put/seqid=0 2024-11-11T07:23:52,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742210_1386 (size=14794) 2024-11-11T07:23:52,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309892448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309892449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309892452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309892457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,464 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309892461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,471 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309892571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309892572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309892572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:23:52,622 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:52,622 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-11T07:23:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:23:52,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:52,625 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:52,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:23:52,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:23:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:52,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:52,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309892792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309892792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309892792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:52,833 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:52,837 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111163234171d7f841748587cebfd28de11f_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163234171d7f841748587cebfd28de11f_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:52,837 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c418213687b1400d9b9cd87452e1d074, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:52,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c418213687b1400d9b9cd87452e1d074 is 175, key is test_row_0/A:col10/1731309832406/Put/seqid=0 2024-11-11T07:23:52,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742211_1387 (size=39749) 2024-11-11T07:23:52,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:23:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:52,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:23:53,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:23:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309893111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:53,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309893112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:53,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309893112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:53,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:23:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c418213687b1400d9b9cd87452e1d074 2024-11-11T07:23:53,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/56f44be5594348bfa19cd1982c44212e is 50, key is test_row_0/B:col10/1731309832406/Put/seqid=0 2024-11-11T07:23:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:23:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742212_1388 (size=12151) 2024-11-11T07:23:53,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:23:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:23:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:53,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309893621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309893621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:53,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309893621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:53,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/56f44be5594348bfa19cd1982c44212e 2024-11-11T07:23:53,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e7fd466a9e73476e83c3344877ac410c is 50, key is test_row_0/C:col10/1731309832406/Put/seqid=0 2024-11-11T07:23:53,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742213_1389 (size=12151) 2024-11-11T07:23:53,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e7fd466a9e73476e83c3344877ac410c 2024-11-11T07:23:53,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c418213687b1400d9b9cd87452e1d074 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c418213687b1400d9b9cd87452e1d074 2024-11-11T07:23:53,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c418213687b1400d9b9cd87452e1d074, entries=200, sequenceid=196, filesize=38.8 K 2024-11-11T07:23:53,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/56f44be5594348bfa19cd1982c44212e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/56f44be5594348bfa19cd1982c44212e 2024-11-11T07:23:53,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/56f44be5594348bfa19cd1982c44212e, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T07:23:53,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/e7fd466a9e73476e83c3344877ac410c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e7fd466a9e73476e83c3344877ac410c 2024-11-11T07:23:53,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e7fd466a9e73476e83c3344877ac410c, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T07:23:53,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b5411b408cdd3e1e96fba71c599080d2 in 1282ms, sequenceid=196, compaction requested=true 2024-11-11T07:23:53,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:53,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:53,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:53,690 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:53,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:53,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:53,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:53,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:53,690 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:53,691 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:53,691 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/A is initiating minor compaction (all files) 2024-11-11T07:23:53,691 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/A in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,691 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/888e0060cd7c4661b7ea31f8492617cc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8c9c426a7e52422daa3f88e8497cb211, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c418213687b1400d9b9cd87452e1d074] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=99.9 K 2024-11-11T07:23:53,691 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,691 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/888e0060cd7c4661b7ea31f8492617cc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8c9c426a7e52422daa3f88e8497cb211, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c418213687b1400d9b9cd87452e1d074] 2024-11-11T07:23:53,691 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:53,691 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/B is initiating minor compaction (all files) 2024-11-11T07:23:53,691 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/B in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,692 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/59bb0e43e8e44cd187affe8660f26d39, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/0c6466af750744b79b99c2f0b9f660e3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/56f44be5594348bfa19cd1982c44212e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=35.9 K 2024-11-11T07:23:53,692 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 59bb0e43e8e44cd187affe8660f26d39, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309830328 2024-11-11T07:23:53,692 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 888e0060cd7c4661b7ea31f8492617cc, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309830328 2024-11-11T07:23:53,692 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c6466af750744b79b99c2f0b9f660e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731309830487 2024-11-11T07:23:53,692 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c9c426a7e52422daa3f88e8497cb211, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731309830487 2024-11-11T07:23:53,692 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f44be5594348bfa19cd1982c44212e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309831766 2024-11-11T07:23:53,693 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c418213687b1400d9b9cd87452e1d074, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309831753 2024-11-11T07:23:53,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:23:53,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,698 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:23:53,700 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#B#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:53,701 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/5d4ad24b27824a3d9cfe4c6c993a0d86 is 50, key is test_row_0/B:col10/1731309832406/Put/seqid=0 2024-11-11T07:23:53,701 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:53,704 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b3ea86184641440c882c90eb29b60778_b5411b408cdd3e1e96fba71c599080d2 store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:53,706 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b3ea86184641440c882c90eb29b60778_b5411b408cdd3e1e96fba71c599080d2, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:53,707 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b3ea86184641440c882c90eb29b60778_b5411b408cdd3e1e96fba71c599080d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:53,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111006564c7940e4d94871471a16c0a8e0e_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309832444/Put/seqid=0 2024-11-11T07:23:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742214_1390 (size=12595) 2024-11-11T07:23:53,719 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/5d4ad24b27824a3d9cfe4c6c993a0d86 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/5d4ad24b27824a3d9cfe4c6c993a0d86 2024-11-11T07:23:53,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742215_1391 (size=4469) 2024-11-11T07:23:53,722 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#A#compaction#333 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:53,722 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7431d4be2cf946bc87c24efb68384517 is 175, key is test_row_0/A:col10/1731309832406/Put/seqid=0 2024-11-11T07:23:53,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742216_1392 (size=12304) 2024-11-11T07:23:53,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:53,726 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/B of b5411b408cdd3e1e96fba71c599080d2 into 5d4ad24b27824a3d9cfe4c6c993a0d86(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:53,726 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:53,726 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/B, priority=13, startTime=1731309833690; duration=0sec 2024-11-11T07:23:53,727 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:53,727 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:B 2024-11-11T07:23:53,727 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:53,736 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:53,737 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/C is initiating minor compaction (all files) 2024-11-11T07:23:53,737 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/C in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:53,737 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d5ea08dda0a94ae1ad8534ed3409c318, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f83ba60b95ba48a697ddeafd8b677df0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e7fd466a9e73476e83c3344877ac410c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=35.9 K 2024-11-11T07:23:53,737 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ea08dda0a94ae1ad8534ed3409c318, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309830328 2024-11-11T07:23:53,738 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f83ba60b95ba48a697ddeafd8b677df0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731309830487 2024-11-11T07:23:53,738 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e7fd466a9e73476e83c3344877ac410c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309831766 2024-11-11T07:23:53,739 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111006564c7940e4d94871471a16c0a8e0e_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111006564c7940e4d94871471a16c0a8e0e_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:53,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c8963ac0b0cb46e38455578eafc5a7a1, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:53,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742217_1393 (size=31549) 2024-11-11T07:23:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c8963ac0b0cb46e38455578eafc5a7a1 is 175, key is test_row_0/A:col10/1731309832444/Put/seqid=0 2024-11-11T07:23:53,750 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7431d4be2cf946bc87c24efb68384517 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7431d4be2cf946bc87c24efb68384517 2024-11-11T07:23:53,755 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:53,755 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/71de7c05715a4ea3ac06dc2e195a9c25 is 50, key is test_row_0/C:col10/1731309832406/Put/seqid=0 2024-11-11T07:23:53,756 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/A of b5411b408cdd3e1e96fba71c599080d2 into 7431d4be2cf946bc87c24efb68384517(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:53,756 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:53,756 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/A, priority=13, startTime=1731309833689; duration=0sec 2024-11-11T07:23:53,757 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:53,757 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:A 2024-11-11T07:23:53,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742218_1394 (size=31105) 2024-11-11T07:23:53,759 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c8963ac0b0cb46e38455578eafc5a7a1 2024-11-11T07:23:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:23:53,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742219_1395 (size=12595) 2024-11-11T07:23:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/a76448df610c4338b699b7d32561f282 is 50, key is test_row_0/B:col10/1731309832444/Put/seqid=0 2024-11-11T07:23:53,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742220_1396 (size=12151) 2024-11-11T07:23:53,770 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/71de7c05715a4ea3ac06dc2e195a9c25 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/71de7c05715a4ea3ac06dc2e195a9c25 2024-11-11T07:23:53,770 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/a76448df610c4338b699b7d32561f282 2024-11-11T07:23:53,777 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/C of b5411b408cdd3e1e96fba71c599080d2 into 71de7c05715a4ea3ac06dc2e195a9c25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:53,777 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:53,777 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/C, priority=13, startTime=1731309833690; duration=0sec 2024-11-11T07:23:53,777 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:53,777 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:C 2024-11-11T07:23:53,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/d6c90ae61e0a43abb859a3d8f1ae7306 is 50, key is test_row_0/C:col10/1731309832444/Put/seqid=0 2024-11-11T07:23:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742221_1397 (size=12151) 2024-11-11T07:23:54,191 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/d6c90ae61e0a43abb859a3d8f1ae7306 2024-11-11T07:23:54,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/c8963ac0b0cb46e38455578eafc5a7a1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c8963ac0b0cb46e38455578eafc5a7a1 2024-11-11T07:23:54,199 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c8963ac0b0cb46e38455578eafc5a7a1, entries=150, sequenceid=208, filesize=30.4 K 2024-11-11T07:23:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/a76448df610c4338b699b7d32561f282 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/a76448df610c4338b699b7d32561f282 2024-11-11T07:23:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,203 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/a76448df610c4338b699b7d32561f282, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T07:23:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/d6c90ae61e0a43abb859a3d8f1ae7306 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d6c90ae61e0a43abb859a3d8f1ae7306 2024-11-11T07:23:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,208 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d6c90ae61e0a43abb859a3d8f1ae7306, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T07:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,208 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for b5411b408cdd3e1e96fba71c599080d2 in 510ms, sequenceid=208, compaction requested=false 2024-11-11T07:23:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:54,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:54,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-11T07:23:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-11T07:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T07:23:54,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5850 sec 2024-11-11T07:23:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.5910 sec 2024-11-11T07:23:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:23:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,772 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,772 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T07:23:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:23:54,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:54,804 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:54,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:23:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:54,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113fbb7d723b324790809c377318fa965d_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309834802/Put/seqid=0 2024-11-11T07:23:54,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742223_1399 (size=27248) 2024-11-11T07:23:54,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309894867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309894875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:23:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309894876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:54,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:54,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:54,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309894982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:54,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309894982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309894991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:23:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309895202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309895202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309895211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:55,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113fbb7d723b324790809c377318fa965d_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113fbb7d723b324790809c377318fa965d_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:55,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7793a43ebe35468e8eba4781cf034568, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:55,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7793a43ebe35468e8eba4781cf034568 is 175, key is test_row_0/A:col10/1731309834802/Put/seqid=0 2024-11-11T07:23:55,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742222_1398 (size=83035) 2024-11-11T07:23:55,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:23:55,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309895511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:55,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309895512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309895531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:55,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7793a43ebe35468e8eba4781cf034568 2024-11-11T07:23:55,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/537769b1ca17466b82875dd73e1661bc is 50, key is test_row_0/B:col10/1731309834802/Put/seqid=0 2024-11-11T07:23:55,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742224_1400 (size=12151) 2024-11-11T07:23:55,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:55,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:23:56,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309896022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:56,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309896031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:56,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/537769b1ca17466b82875dd73e1661bc 2024-11-11T07:23:56,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/92b12d04e056422b8235234b1811d6d5 is 50, key is test_row_0/C:col10/1731309834802/Put/seqid=0 2024-11-11T07:23:56,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309896051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:56,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742225_1401 (size=12151) 2024-11-11T07:23:56,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:56,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/92b12d04e056422b8235234b1811d6d5 2024-11-11T07:23:56,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/7793a43ebe35468e8eba4781cf034568 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7793a43ebe35468e8eba4781cf034568 2024-11-11T07:23:56,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7793a43ebe35468e8eba4781cf034568, entries=450, sequenceid=223, filesize=81.1 K 2024-11-11T07:23:56,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/537769b1ca17466b82875dd73e1661bc as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/537769b1ca17466b82875dd73e1661bc 2024-11-11T07:23:56,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/537769b1ca17466b82875dd73e1661bc, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T07:23:56,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/92b12d04e056422b8235234b1811d6d5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/92b12d04e056422b8235234b1811d6d5 2024-11-11T07:23:56,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/92b12d04e056422b8235234b1811d6d5, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T07:23:56,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b5411b408cdd3e1e96fba71c599080d2 in 1677ms, sequenceid=223, compaction requested=true 2024-11-11T07:23:56,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:56,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:56,482 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:56,482 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:56,482 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 145689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:56,482 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:56,482 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/A is initiating minor compaction (all files) 2024-11-11T07:23:56,482 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/B is initiating minor compaction (all files) 2024-11-11T07:23:56,482 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/A in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,482 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/B in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,483 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7431d4be2cf946bc87c24efb68384517, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c8963ac0b0cb46e38455578eafc5a7a1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7793a43ebe35468e8eba4781cf034568] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=142.3 K 2024-11-11T07:23:56,483 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/5d4ad24b27824a3d9cfe4c6c993a0d86, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/a76448df610c4338b699b7d32561f282, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/537769b1ca17466b82875dd73e1661bc] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=36.0 K 2024-11-11T07:23:56,483 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,483 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7431d4be2cf946bc87c24efb68384517, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c8963ac0b0cb46e38455578eafc5a7a1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7793a43ebe35468e8eba4781cf034568] 2024-11-11T07:23:56,483 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4ad24b27824a3d9cfe4c6c993a0d86, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309831766 2024-11-11T07:23:56,483 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7431d4be2cf946bc87c24efb68384517, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309831766 2024-11-11T07:23:56,483 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a76448df610c4338b699b7d32561f282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731309832444 2024-11-11T07:23:56,483 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8963ac0b0cb46e38455578eafc5a7a1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731309832444 2024-11-11T07:23:56,483 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7793a43ebe35468e8eba4781cf034568, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731309834725 2024-11-11T07:23:56,484 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 537769b1ca17466b82875dd73e1661bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731309834728 2024-11-11T07:23:56,490 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:56,491 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110a8d70c99c5b408a9f03a7abe431900c_b5411b408cdd3e1e96fba71c599080d2 store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:56,491 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:56,492 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/452ad87502494fca8fd9622f82b6b0d7 is 50, key is test_row_0/B:col10/1731309834802/Put/seqid=0 2024-11-11T07:23:56,494 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110a8d70c99c5b408a9f03a7abe431900c_b5411b408cdd3e1e96fba71c599080d2, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:56,494 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a8d70c99c5b408a9f03a7abe431900c_b5411b408cdd3e1e96fba71c599080d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:56,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742226_1402 (size=12697) 2024-11-11T07:23:56,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742227_1403 (size=4469) 2024-11-11T07:23:56,504 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/452ad87502494fca8fd9622f82b6b0d7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/452ad87502494fca8fd9622f82b6b0d7 2024-11-11T07:23:56,509 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/B of b5411b408cdd3e1e96fba71c599080d2 into 452ad87502494fca8fd9622f82b6b0d7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:56,509 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:56,509 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/B, priority=13, startTime=1731309836482; duration=0sec 2024-11-11T07:23:56,509 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:56,509 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:B 2024-11-11T07:23:56,509 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:56,510 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:56,510 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/C is initiating minor compaction (all files) 2024-11-11T07:23:56,510 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/C in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,510 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/71de7c05715a4ea3ac06dc2e195a9c25, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d6c90ae61e0a43abb859a3d8f1ae7306, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/92b12d04e056422b8235234b1811d6d5] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=36.0 K 2024-11-11T07:23:56,511 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 71de7c05715a4ea3ac06dc2e195a9c25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731309831766 2024-11-11T07:23:56,511 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c90ae61e0a43abb859a3d8f1ae7306, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731309832444 2024-11-11T07:23:56,511 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b12d04e056422b8235234b1811d6d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731309834728 2024-11-11T07:23:56,528 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#C#compaction#343 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:56,528 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/7d6be24a13664e2d9cc8aa838d8a22ca is 50, key is test_row_0/C:col10/1731309834802/Put/seqid=0 2024-11-11T07:23:56,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742228_1404 (size=12697) 2024-11-11T07:23:56,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:23:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:56,613 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:23:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d2e3b54a0edb4d0da392069c4077b680_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309834874/Put/seqid=0 2024-11-11T07:23:56,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742229_1405 (size=12304) 2024-11-11T07:23:56,901 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#A#compaction#341 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:56,901 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/bf6abc9984e64e628d48c6b5de06e992 is 175, key is test_row_0/A:col10/1731309834802/Put/seqid=0 2024-11-11T07:23:56,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742230_1406 (size=31651) 2024-11-11T07:23:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:23:56,942 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/7d6be24a13664e2d9cc8aa838d8a22ca as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7d6be24a13664e2d9cc8aa838d8a22ca 2024-11-11T07:23:56,946 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/C of b5411b408cdd3e1e96fba71c599080d2 into 7d6be24a13664e2d9cc8aa838d8a22ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:56,946 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:56,946 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/C, priority=13, startTime=1731309836482; duration=0sec 2024-11-11T07:23:56,946 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:56,946 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:C 2024-11-11T07:23:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:57,029 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d2e3b54a0edb4d0da392069c4077b680_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d2e3b54a0edb4d0da392069c4077b680_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/86a1ecdbc661440f9f0476757b018949, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/86a1ecdbc661440f9f0476757b018949 is 175, key is test_row_0/A:col10/1731309834874/Put/seqid=0 2024-11-11T07:23:57,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742231_1407 (size=31105) 2024-11-11T07:23:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:57,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309897071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309897072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,080 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309897073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309897191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309897192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,310 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/bf6abc9984e64e628d48c6b5de06e992 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/bf6abc9984e64e628d48c6b5de06e992 2024-11-11T07:23:57,318 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/A of b5411b408cdd3e1e96fba71c599080d2 into bf6abc9984e64e628d48c6b5de06e992(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:57,318 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:57,318 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/A, priority=13, startTime=1731309836481; duration=0sec 2024-11-11T07:23:57,318 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:57,318 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:A 2024-11-11T07:23:57,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309897411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309897412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,435 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/86a1ecdbc661440f9f0476757b018949 2024-11-11T07:23:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/82b71419786c4d9f9fd5bb3bbc58e8e2 is 50, key is test_row_0/B:col10/1731309834874/Put/seqid=0 2024-11-11T07:23:57,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742232_1408 (size=12151) 2024-11-11T07:23:57,450 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/82b71419786c4d9f9fd5bb3bbc58e8e2 2024-11-11T07:23:57,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/5c765f8c68124537b5ab6caf707f7fc8 is 50, key is test_row_0/C:col10/1731309834874/Put/seqid=0 2024-11-11T07:23:57,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742233_1409 (size=12151) 2024-11-11T07:23:57,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309897722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309897731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:57,862 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/5c765f8c68124537b5ab6caf707f7fc8 2024-11-11T07:23:57,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/86a1ecdbc661440f9f0476757b018949 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/86a1ecdbc661440f9f0476757b018949 2024-11-11T07:23:57,872 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/86a1ecdbc661440f9f0476757b018949, entries=150, sequenceid=248, filesize=30.4 K 2024-11-11T07:23:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/82b71419786c4d9f9fd5bb3bbc58e8e2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/82b71419786c4d9f9fd5bb3bbc58e8e2 2024-11-11T07:23:57,877 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/82b71419786c4d9f9fd5bb3bbc58e8e2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T07:23:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/5c765f8c68124537b5ab6caf707f7fc8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/5c765f8c68124537b5ab6caf707f7fc8 2024-11-11T07:23:57,881 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/5c765f8c68124537b5ab6caf707f7fc8, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T07:23:57,882 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b5411b408cdd3e1e96fba71c599080d2 in 1269ms, sequenceid=248, compaction requested=false 2024-11-11T07:23:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T07:23:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T07:23:57,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T07:23:57,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1100 sec 2024-11-11T07:23:57,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.1170 sec 2024-11-11T07:23:58,262 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:58,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:23:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:58,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e64ad5d029944188e9071b64dd381c8_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309838251/Put/seqid=0 2024-11-11T07:23:58,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742234_1410 (size=14944) 2024-11-11T07:23:58,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:58,403 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e64ad5d029944188e9071b64dd381c8_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e64ad5d029944188e9071b64dd381c8_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:58,404 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/0ce37424f59e452e9623cb024f5786fa, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:58,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/0ce37424f59e452e9623cb024f5786fa is 175, key is test_row_0/A:col10/1731309838251/Put/seqid=0 2024-11-11T07:23:58,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742235_1411 (size=39899) 2024-11-11T07:23:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309898555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:58,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309898558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309898701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309898711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:58,861 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/0ce37424f59e452e9623cb024f5786fa 2024-11-11T07:23:58,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/03be6587761443f38108c11e35ead0eb is 50, key is test_row_0/B:col10/1731309838251/Put/seqid=0 2024-11-11T07:23:58,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:23:58,942 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:23:58,942 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:23:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T07:23:58,945 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:23:58,946 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:23:58,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:23:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:23:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309898932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:58,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309898932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:59,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742236_1412 (size=12301) 2024-11-11T07:23:59,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/03be6587761443f38108c11e35ead0eb 2024-11-11T07:23:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:23:59,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/681d73ae1efe4c7399d976a34d387fad is 50, key is test_row_0/C:col10/1731309838251/Put/seqid=0 2024-11-11T07:23:59,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:23:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309899101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:59,119 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:23:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:59,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742237_1413 (size=12301) 2024-11-11T07:23:59,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/681d73ae1efe4c7399d976a34d387fad 2024-11-11T07:23:59,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/0ce37424f59e452e9623cb024f5786fa as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/0ce37424f59e452e9623cb024f5786fa 2024-11-11T07:23:59,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/0ce37424f59e452e9623cb024f5786fa, entries=200, sequenceid=263, filesize=39.0 K 2024-11-11T07:23:59,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/03be6587761443f38108c11e35ead0eb as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/03be6587761443f38108c11e35ead0eb 2024-11-11T07:23:59,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/03be6587761443f38108c11e35ead0eb, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T07:23:59,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/681d73ae1efe4c7399d976a34d387fad as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/681d73ae1efe4c7399d976a34d387fad 2024-11-11T07:23:59,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/681d73ae1efe4c7399d976a34d387fad, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T07:23:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:23:59,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:23:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:59,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b5411b408cdd3e1e96fba71c599080d2 in 1004ms, sequenceid=263, compaction requested=true 2024-11-11T07:23:59,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:59,266 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:23:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:59,267 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:23:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:23:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:59,268 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:59,268 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/A is initiating minor compaction (all files) 2024-11-11T07:23:59,268 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/A in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,268 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/bf6abc9984e64e628d48c6b5de06e992, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/86a1ecdbc661440f9f0476757b018949, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/0ce37424f59e452e9623cb024f5786fa] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=100.2 K 2024-11-11T07:23:59,268 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,269 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/bf6abc9984e64e628d48c6b5de06e992, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/86a1ecdbc661440f9f0476757b018949, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/0ce37424f59e452e9623cb024f5786fa] 2024-11-11T07:23:59,269 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6abc9984e64e628d48c6b5de06e992, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731309834728 2024-11-11T07:23:59,269 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:59,269 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/B is initiating minor compaction (all files) 2024-11-11T07:23:59,269 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/B in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,269 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/452ad87502494fca8fd9622f82b6b0d7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/82b71419786c4d9f9fd5bb3bbc58e8e2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/03be6587761443f38108c11e35ead0eb] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=36.3 K 2024-11-11T07:23:59,270 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a1ecdbc661440f9f0476757b018949, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731309834856 2024-11-11T07:23:59,270 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 452ad87502494fca8fd9622f82b6b0d7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731309834728 2024-11-11T07:23:59,270 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ce37424f59e452e9623cb024f5786fa, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309837056 2024-11-11T07:23:59,271 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 82b71419786c4d9f9fd5bb3bbc58e8e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731309834856 2024-11-11T07:23:59,271 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 03be6587761443f38108c11e35ead0eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309837067 2024-11-11T07:23:59,292 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:59,298 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:59,299 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/16ef86fb7b7b4584a567240e1544aec5 is 50, key is test_row_0/B:col10/1731309838251/Put/seqid=0 2024-11-11T07:23:59,340 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111fae5ebfe08e3451ca3a5e277e83af572_b5411b408cdd3e1e96fba71c599080d2 store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:59,342 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111fae5ebfe08e3451ca3a5e277e83af572_b5411b408cdd3e1e96fba71c599080d2, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:59,343 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fae5ebfe08e3451ca3a5e277e83af572_b5411b408cdd3e1e96fba71c599080d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:59,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:23:59,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:23:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:59,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:23:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742238_1414 (size=12949) 2024-11-11T07:23:59,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111314c49fe3a7b4fa7ac860c4ea5aa1593_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309838502/Put/seqid=0 2024-11-11T07:23:59,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742239_1415 (size=4469) 2024-11-11T07:23:59,508 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#A#compaction#350 average throughput is 0.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:59,509 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/2f81728c46154279bfeef73d8e3ca69f is 175, key is test_row_0/A:col10/1731309838251/Put/seqid=0 2024-11-11T07:23:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:23:59,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:23:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309899571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:59,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309899580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:59,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742240_1416 (size=14994) 2024-11-11T07:23:59,648 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:23:59,656 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111314c49fe3a7b4fa7ac860c4ea5aa1593_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111314c49fe3a7b4fa7ac860c4ea5aa1593_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:23:59,660 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/779662186d694e46921d89917e62125d, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:23:59,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/779662186d694e46921d89917e62125d is 175, key is test_row_0/A:col10/1731309838502/Put/seqid=0 2024-11-11T07:23:59,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742241_1417 (size=31903) 2024-11-11T07:23:59,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:59,717 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/2f81728c46154279bfeef73d8e3ca69f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/2f81728c46154279bfeef73d8e3ca69f 2024-11-11T07:23:59,723 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/A of b5411b408cdd3e1e96fba71c599080d2 into 2f81728c46154279bfeef73d8e3ca69f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:59,723 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:59,723 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/A, priority=13, startTime=1731309839266; duration=0sec 2024-11-11T07:23:59,723 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:23:59,723 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:A 2024-11-11T07:23:59,723 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:23:59,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:23:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:23:59,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/C is initiating minor compaction (all files) 2024-11-11T07:23:59,729 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/C in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,729 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7d6be24a13664e2d9cc8aa838d8a22ca, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/5c765f8c68124537b5ab6caf707f7fc8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/681d73ae1efe4c7399d976a34d387fad] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=36.3 K 2024-11-11T07:23:59,729 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d6be24a13664e2d9cc8aa838d8a22ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731309834728 2024-11-11T07:23:59,730 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c765f8c68124537b5ab6caf707f7fc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731309834856 2024-11-11T07:23:59,730 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681d73ae1efe4c7399d976a34d387fad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309837067 2024-11-11T07:23:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309899696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:59,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:59,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742242_1418 (size=39949) 2024-11-11T07:23:59,762 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#C#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:23:59,762 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/c5bebebc776849a3860687ae8514d28e is 50, key is test_row_0/C:col10/1731309838251/Put/seqid=0 2024-11-11T07:23:59,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/779662186d694e46921d89917e62125d 2024-11-11T07:23:59,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:23:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309899717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:23:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:23:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:23:59,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742243_1419 (size=12949) 2024-11-11T07:23:59,849 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/c5bebebc776849a3860687ae8514d28e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c5bebebc776849a3860687ae8514d28e 2024-11-11T07:23:59,854 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/C of b5411b408cdd3e1e96fba71c599080d2 into c5bebebc776849a3860687ae8514d28e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:59,854 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:59,854 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/C, priority=13, startTime=1731309839267; duration=0sec 2024-11-11T07:23:59,854 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:59,855 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:C 2024-11-11T07:23:59,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/6ca9c4bc83b24e199b34c7d3d733abe9 is 50, key is test_row_0/B:col10/1731309838502/Put/seqid=0 2024-11-11T07:23:59,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:23:59,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:23:59,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:23:59,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:23:59,912 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/16ef86fb7b7b4584a567240e1544aec5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/16ef86fb7b7b4584a567240e1544aec5 2024-11-11T07:23:59,918 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/B of b5411b408cdd3e1e96fba71c599080d2 into 16ef86fb7b7b4584a567240e1544aec5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:23:59,918 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:23:59,918 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/B, priority=13, startTime=1731309839267; duration=0sec 2024-11-11T07:23:59,918 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:23:59,918 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:B 2024-11-11T07:24:00,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:24:00,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:24:00,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:24:00,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309899941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:00,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309899972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:00,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:24:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:24:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742244_1420 (size=12301) 2024-11-11T07:24:00,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/6ca9c4bc83b24e199b34c7d3d733abe9 2024-11-11T07:24:00,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/42e901681cc5418283fe6708468a39ca is 50, key is test_row_0/C:col10/1731309838502/Put/seqid=0 2024-11-11T07:24:00,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:24:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:24:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309900432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:00,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742245_1421 (size=12301) 2024-11-11T07:24:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309900432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:00,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:24:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:24:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:24:00,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:24:00,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:24:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:24:00,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/42e901681cc5418283fe6708468a39ca 2024-11-11T07:24:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:00,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:00,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/779662186d694e46921d89917e62125d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/779662186d694e46921d89917e62125d 2024-11-11T07:24:00,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/779662186d694e46921d89917e62125d, entries=200, sequenceid=287, filesize=39.0 K 2024-11-11T07:24:00,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/6ca9c4bc83b24e199b34c7d3d733abe9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/6ca9c4bc83b24e199b34c7d3d733abe9 2024-11-11T07:24:00,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/6ca9c4bc83b24e199b34c7d3d733abe9, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T07:24:00,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/42e901681cc5418283fe6708468a39ca as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/42e901681cc5418283fe6708468a39ca 2024-11-11T07:24:00,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/42e901681cc5418283fe6708468a39ca, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T07:24:00,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b5411b408cdd3e1e96fba71c599080d2 in 1558ms, sequenceid=287, compaction requested=false 2024-11-11T07:24:00,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:24:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:24:01,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:24:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:01,140 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:24:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:01,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. as already flushing 2024-11-11T07:24:01,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111df470b18f6dc46349c6e9c6cfef78ec8_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309839575/Put/seqid=0 2024-11-11T07:24:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742246_1422 (size=14994) 2024-11-11T07:24:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:01,442 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111df470b18f6dc46349c6e9c6cfef78ec8_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df470b18f6dc46349c6e9c6cfef78ec8_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/f784e7efd85849b8afbee72289f81636, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/f784e7efd85849b8afbee72289f81636 is 175, key is test_row_0/A:col10/1731309839575/Put/seqid=0 2024-11-11T07:24:01,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742247_1423 (size=39949) 2024-11-11T07:24:01,549 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/f784e7efd85849b8afbee72289f81636 2024-11-11T07:24:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/b50e769779184a4982a92cdd16e85df9 is 50, key is test_row_0/B:col10/1731309839575/Put/seqid=0 2024-11-11T07:24:01,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742248_1424 (size=12301) 2024-11-11T07:24:01,837 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/b50e769779184a4982a92cdd16e85df9 2024-11-11T07:24:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/f1a30ec0c0e74741a4e87305f447cd3d is 50, key is test_row_0/C:col10/1731309839575/Put/seqid=0 2024-11-11T07:24:01,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742249_1425 (size=12301) 2024-11-11T07:24:01,928 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/f1a30ec0c0e74741a4e87305f447cd3d 2024-11-11T07:24:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/f784e7efd85849b8afbee72289f81636 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/f784e7efd85849b8afbee72289f81636 2024-11-11T07:24:01,938 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/f784e7efd85849b8afbee72289f81636, entries=200, sequenceid=303, filesize=39.0 K 2024-11-11T07:24:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/b50e769779184a4982a92cdd16e85df9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/b50e769779184a4982a92cdd16e85df9 2024-11-11T07:24:01,953 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/b50e769779184a4982a92cdd16e85df9, entries=150, sequenceid=303, filesize=12.0 K 2024-11-11T07:24:01,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/f1a30ec0c0e74741a4e87305f447cd3d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f1a30ec0c0e74741a4e87305f447cd3d 2024-11-11T07:24:01,969 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f1a30ec0c0e74741a4e87305f447cd3d, entries=150, sequenceid=303, filesize=12.0 K 2024-11-11T07:24:01,970 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=114.05 KB/116790 for b5411b408cdd3e1e96fba71c599080d2 in 829ms, sequenceid=303, compaction requested=true 2024-11-11T07:24:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:24:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T07:24:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T07:24:01,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T07:24:01,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0310 sec 2024-11-11T07:24:01,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.0370 sec 2024-11-11T07:24:02,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:02,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T07:24:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:02,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113a13ad71a9fb4d61abd6b33bb9d5017b_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309841917/Put/seqid=0 2024-11-11T07:24:02,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742250_1426 (size=14994) 2024-11-11T07:24:02,137 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:02,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113a13ad71a9fb4d61abd6b33bb9d5017b_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113a13ad71a9fb4d61abd6b33bb9d5017b_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:02,150 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/83165fb770754091bb8f258461a5f8c8, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:02,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/83165fb770754091bb8f258461a5f8c8 is 175, key is test_row_0/A:col10/1731309841917/Put/seqid=0 2024-11-11T07:24:02,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742251_1427 (size=39949) 2024-11-11T07:24:02,480 INFO [AsyncFSWAL-0-hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537-prefix:f334e8b0fbdf,39329,1731309728191 {}] wal.AbstractFSWAL(1368): Slow sync cost: 152 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40127,DS-74a6e686-2d71-4357-ae40-76976cd3e84c,DISK]] 2024-11-11T07:24:02,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309902524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309902526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309902533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:02,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35268 deadline: 1731309902577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:02,716 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18475 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:02,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/83165fb770754091bb8f258461a5f8c8 2024-11-11T07:24:02,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/22db1880d9f740bb9bbf5c45c47b4b9e is 50, key is test_row_0/B:col10/1731309841917/Put/seqid=0 2024-11-11T07:24:02,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309902742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:02,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:02,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309902761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:02,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309902764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742252_1428 (size=12301) 2024-11-11T07:24:02,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/22db1880d9f740bb9bbf5c45c47b4b9e 2024-11-11T07:24:02,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/34bfefec07ca462281c29b0b21a1369c is 50, key is test_row_0/C:col10/1731309841917/Put/seqid=0 2024-11-11T07:24:03,047 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-11T07:24:03,047 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,047 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,047 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,084 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-11T07:24:03,084 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,084 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,085 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35238 deadline: 1731309903062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35286 deadline: 1731309903062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35210 deadline: 1731309903062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:03,102 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-11T07:24:03,102 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,102 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,102 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:24:03,112 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:03,113 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-11T07:24:03,113 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-11T07:24:03,113 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,113 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,113 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,113 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,114 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,114 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742253_1429 (size=12301) 2024-11-11T07:24:03,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/34bfefec07ca462281c29b0b21a1369c 2024-11-11T07:24:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/83165fb770754091bb8f258461a5f8c8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/83165fb770754091bb8f258461a5f8c8 2024-11-11T07:24:03,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/83165fb770754091bb8f258461a5f8c8, entries=200, sequenceid=324, filesize=39.0 K 2024-11-11T07:24:03,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/22db1880d9f740bb9bbf5c45c47b4b9e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/22db1880d9f740bb9bbf5c45c47b4b9e 2024-11-11T07:24:03,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35260 deadline: 1731309903161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:03,163 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in b5411b408cdd3e1e96fba71c599080d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:03,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/22db1880d9f740bb9bbf5c45c47b4b9e, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T07:24:03,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/34bfefec07ca462281c29b0b21a1369c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/34bfefec07ca462281c29b0b21a1369c 2024-11-11T07:24:03,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/34bfefec07ca462281c29b0b21a1369c, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T07:24:03,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b5411b408cdd3e1e96fba71c599080d2 in 1195ms, sequenceid=324, compaction requested=true 2024-11-11T07:24:03,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:24:03,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:03,221 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:03,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:03,221 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:03,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:03,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:03,223 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:03,223 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/A is initiating minor compaction (all files) 2024-11-11T07:24:03,223 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/A in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:03,223 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/2f81728c46154279bfeef73d8e3ca69f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/779662186d694e46921d89917e62125d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/f784e7efd85849b8afbee72289f81636, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/83165fb770754091bb8f258461a5f8c8] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=148.2 K 2024-11-11T07:24:03,223 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:03,223 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/2f81728c46154279bfeef73d8e3ca69f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/779662186d694e46921d89917e62125d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/f784e7efd85849b8afbee72289f81636, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/83165fb770754091bb8f258461a5f8c8] 2024-11-11T07:24:03,223 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:03,223 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/B is initiating minor compaction (all files) 2024-11-11T07:24:03,223 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/B in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:03,223 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/16ef86fb7b7b4584a567240e1544aec5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/6ca9c4bc83b24e199b34c7d3d733abe9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/b50e769779184a4982a92cdd16e85df9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/22db1880d9f740bb9bbf5c45c47b4b9e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=48.7 K 2024-11-11T07:24:03,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5411b408cdd3e1e96fba71c599080d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:03,224 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f81728c46154279bfeef73d8e3ca69f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309837067 2024-11-11T07:24:03,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:03,224 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ef86fb7b7b4584a567240e1544aec5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309837067 2024-11-11T07:24:03,224 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca9c4bc83b24e199b34c7d3d733abe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309838502 2024-11-11T07:24:03,225 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 779662186d694e46921d89917e62125d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309838502 2024-11-11T07:24:03,225 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b50e769779184a4982a92cdd16e85df9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731309839537 2024-11-11T07:24:03,225 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting f784e7efd85849b8afbee72289f81636, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731309839430 2024-11-11T07:24:03,225 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 22db1880d9f740bb9bbf5c45c47b4b9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731309841917 2024-11-11T07:24:03,225 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83165fb770754091bb8f258461a5f8c8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731309841917 2024-11-11T07:24:03,259 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#B#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:03,259 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/962094d0ff8e44cd85d0b2461f9a94cf is 50, key is test_row_0/B:col10/1731309841917/Put/seqid=0 2024-11-11T07:24:03,369 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:03,406 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:03,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T07:24:03,406 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-11T07:24:03,406 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-11T07:24:03,406 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,406 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,406 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,406 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,406 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,406 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,427 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-11T07:24:03,427 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:03,427 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,427 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:03,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:03,478 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ee1d1dd7afb94e5a9f7dfeca7e24ee8e_b5411b408cdd3e1e96fba71c599080d2 store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:03,483 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ee1d1dd7afb94e5a9f7dfeca7e24ee8e_b5411b408cdd3e1e96fba71c599080d2, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:03,484 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ee1d1dd7afb94e5a9f7dfeca7e24ee8e_b5411b408cdd3e1e96fba71c599080d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:03,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b1644b74812e4f9a892229353c52a36b_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_0/A:col10/1731309843404/Put/seqid=0 2024-11-11T07:24:03,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742254_1430 (size=13085) 2024-11-11T07:24:03,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742255_1431 (size=4469) 2024-11-11T07:24:03,650 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#A#compaction#363 average throughput is 0.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:03,650 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/b2274637e0f94e95974659de28f8b5d2 is 175, key is test_row_0/A:col10/1731309841917/Put/seqid=0 2024-11-11T07:24:03,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742256_1432 (size=12454) 2024-11-11T07:24:03,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742257_1433 (size=32039) 2024-11-11T07:24:03,910 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/b2274637e0f94e95974659de28f8b5d2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/b2274637e0f94e95974659de28f8b5d2 2024-11-11T07:24:03,915 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/A of b5411b408cdd3e1e96fba71c599080d2 into b2274637e0f94e95974659de28f8b5d2(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:03,915 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:24:03,915 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/A, priority=12, startTime=1731309843221; duration=0sec 2024-11-11T07:24:03,916 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:03,916 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:A 2024-11-11T07:24:03,916 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:03,920 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:03,920 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): b5411b408cdd3e1e96fba71c599080d2/C is initiating minor compaction (all files) 2024-11-11T07:24:03,921 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b5411b408cdd3e1e96fba71c599080d2/C in TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:03,921 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c5bebebc776849a3860687ae8514d28e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/42e901681cc5418283fe6708468a39ca, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f1a30ec0c0e74741a4e87305f447cd3d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/34bfefec07ca462281c29b0b21a1369c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp, totalSize=48.7 K 2024-11-11T07:24:03,921 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5bebebc776849a3860687ae8514d28e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731309837067 2024-11-11T07:24:03,922 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42e901681cc5418283fe6708468a39ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731309838502 2024-11-11T07:24:03,922 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a30ec0c0e74741a4e87305f447cd3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731309839537 2024-11-11T07:24:03,922 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34bfefec07ca462281c29b0b21a1369c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731309841917 2024-11-11T07:24:03,972 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5411b408cdd3e1e96fba71c599080d2#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:03,973 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/7974338ae739441ba4d482a483463e3c is 50, key is test_row_0/C:col10/1731309841917/Put/seqid=0 2024-11-11T07:24:04,021 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/962094d0ff8e44cd85d0b2461f9a94cf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/962094d0ff8e44cd85d0b2461f9a94cf 2024-11-11T07:24:04,026 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/B of b5411b408cdd3e1e96fba71c599080d2 into 962094d0ff8e44cd85d0b2461f9a94cf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:04,026 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:24:04,026 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/B, priority=12, startTime=1731309843221; duration=0sec 2024-11-11T07:24:04,027 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:04,027 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:B 2024-11-11T07:24:04,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742258_1434 (size=13085) 2024-11-11T07:24:04,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:04,169 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b1644b74812e4f9a892229353c52a36b_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b1644b74812e4f9a892229353c52a36b_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:04,170 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8e590a2fb39642d58b3a7ae08144747f, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:04,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8e590a2fb39642d58b3a7ae08144747f is 175, key is test_row_0/A:col10/1731309843404/Put/seqid=0 2024-11-11T07:24:04,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742259_1435 (size=31255) 2024-11-11T07:24:04,295 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8e590a2fb39642d58b3a7ae08144747f 2024-11-11T07:24:04,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/9907cf43bd6b4f568521bb923f32c8b1 is 50, key is test_row_0/B:col10/1731309843404/Put/seqid=0 2024-11-11T07:24:04,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742260_1436 (size=12301) 2024-11-11T07:24:04,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/9907cf43bd6b4f568521bb923f32c8b1 2024-11-11T07:24:04,575 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/7974338ae739441ba4d482a483463e3c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7974338ae739441ba4d482a483463e3c 2024-11-11T07:24:04,585 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b5411b408cdd3e1e96fba71c599080d2/C of b5411b408cdd3e1e96fba71c599080d2 into 7974338ae739441ba4d482a483463e3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:04,585 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:24:04,585 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., storeName=b5411b408cdd3e1e96fba71c599080d2/C, priority=12, startTime=1731309843224; duration=0sec 2024-11-11T07:24:04,585 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:04,585 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5411b408cdd3e1e96fba71c599080d2:C 2024-11-11T07:24:04,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/faf0983e2d424cf8bb1145083f871863 is 50, key is test_row_0/C:col10/1731309843404/Put/seqid=0 2024-11-11T07:24:04,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742261_1437 (size=12301) 2024-11-11T07:24:04,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/faf0983e2d424cf8bb1145083f871863 2024-11-11T07:24:04,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/8e590a2fb39642d58b3a7ae08144747f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8e590a2fb39642d58b3a7ae08144747f 2024-11-11T07:24:04,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8e590a2fb39642d58b3a7ae08144747f, entries=150, sequenceid=341, filesize=30.5 K 2024-11-11T07:24:04,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/9907cf43bd6b4f568521bb923f32c8b1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/9907cf43bd6b4f568521bb923f32c8b1 2024-11-11T07:24:04,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/9907cf43bd6b4f568521bb923f32c8b1, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T07:24:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/faf0983e2d424cf8bb1145083f871863 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/faf0983e2d424cf8bb1145083f871863 2024-11-11T07:24:04,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/faf0983e2d424cf8bb1145083f871863, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T07:24:04,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=6.71 KB/6870 for b5411b408cdd3e1e96fba71c599080d2 in 1332ms, sequenceid=341, compaction requested=false 2024-11-11T07:24:04,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b5411b408cdd3e1e96fba71c599080d2: 2024-11-11T07:24:06,528 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:24:10,990 WARN [regionserver/f334e8b0fbdf:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-11T07:24:10,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/a48d037e1eca4220b93ffb616adda526, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/4257ceaa9d174c5a829634d2a79af2cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/516fff8944ca4de3a1872b7c2f420144, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/001c78b3d658479f8beeca4ffb2591bf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7bcfcc3e0b24439bbe044f2195655f5d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7de5357a61d34d9894d21b325de56f71, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/d8392e9eef884b1195c0907906939c25, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/94b3c1ea444942bab47abbf35a9b90e5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/25337c77546e450e834af2ead537bee0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/015ae56fb0874a5e9e20f5633a688769, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/888e0060cd7c4661b7ea31f8492617cc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8c9c426a7e52422daa3f88e8497cb211, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c418213687b1400d9b9cd87452e1d074, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7431d4be2cf946bc87c24efb68384517, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c8963ac0b0cb46e38455578eafc5a7a1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7793a43ebe35468e8eba4781cf034568, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/bf6abc9984e64e628d48c6b5de06e992, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/86a1ecdbc661440f9f0476757b018949, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/0ce37424f59e452e9623cb024f5786fa, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/2f81728c46154279bfeef73d8e3ca69f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/779662186d694e46921d89917e62125d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/f784e7efd85849b8afbee72289f81636, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/83165fb770754091bb8f258461a5f8c8] to archive 2024-11-11T07:24:11,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:24:11,073 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/a48d037e1eca4220b93ffb616adda526 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/a48d037e1eca4220b93ffb616adda526 2024-11-11T07:24:11,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/4257ceaa9d174c5a829634d2a79af2cd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/4257ceaa9d174c5a829634d2a79af2cd 2024-11-11T07:24:11,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/516fff8944ca4de3a1872b7c2f420144 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/516fff8944ca4de3a1872b7c2f420144 2024-11-11T07:24:11,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/001c78b3d658479f8beeca4ffb2591bf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/001c78b3d658479f8beeca4ffb2591bf 2024-11-11T07:24:11,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7bcfcc3e0b24439bbe044f2195655f5d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7bcfcc3e0b24439bbe044f2195655f5d 2024-11-11T07:24:11,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7de5357a61d34d9894d21b325de56f71 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7de5357a61d34d9894d21b325de56f71 2024-11-11T07:24:11,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/d8392e9eef884b1195c0907906939c25 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/d8392e9eef884b1195c0907906939c25 2024-11-11T07:24:11,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/94b3c1ea444942bab47abbf35a9b90e5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/94b3c1ea444942bab47abbf35a9b90e5 2024-11-11T07:24:11,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/25337c77546e450e834af2ead537bee0 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/25337c77546e450e834af2ead537bee0 2024-11-11T07:24:11,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/015ae56fb0874a5e9e20f5633a688769 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/015ae56fb0874a5e9e20f5633a688769 2024-11-11T07:24:11,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/888e0060cd7c4661b7ea31f8492617cc to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/888e0060cd7c4661b7ea31f8492617cc 2024-11-11T07:24:11,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8c9c426a7e52422daa3f88e8497cb211 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8c9c426a7e52422daa3f88e8497cb211 2024-11-11T07:24:11,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c418213687b1400d9b9cd87452e1d074 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c418213687b1400d9b9cd87452e1d074 2024-11-11T07:24:11,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7431d4be2cf946bc87c24efb68384517 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7431d4be2cf946bc87c24efb68384517 2024-11-11T07:24:11,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c8963ac0b0cb46e38455578eafc5a7a1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/c8963ac0b0cb46e38455578eafc5a7a1 2024-11-11T07:24:11,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7793a43ebe35468e8eba4781cf034568 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/7793a43ebe35468e8eba4781cf034568 2024-11-11T07:24:11,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/bf6abc9984e64e628d48c6b5de06e992 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/bf6abc9984e64e628d48c6b5de06e992 2024-11-11T07:24:11,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/86a1ecdbc661440f9f0476757b018949 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/86a1ecdbc661440f9f0476757b018949 2024-11-11T07:24:11,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/0ce37424f59e452e9623cb024f5786fa to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/0ce37424f59e452e9623cb024f5786fa 2024-11-11T07:24:11,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/2f81728c46154279bfeef73d8e3ca69f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/2f81728c46154279bfeef73d8e3ca69f 2024-11-11T07:24:11,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/779662186d694e46921d89917e62125d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/779662186d694e46921d89917e62125d 2024-11-11T07:24:11,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/f784e7efd85849b8afbee72289f81636 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/f784e7efd85849b8afbee72289f81636 2024-11-11T07:24:11,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/83165fb770754091bb8f258461a5f8c8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/83165fb770754091bb8f258461a5f8c8 2024-11-11T07:24:11,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/daad491a0aae4780919ea70ec9330274, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/4cc79c4c1000423aaac1c82183c841a4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/857a063be7d94fb3bab81d5f53c3948e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/25fb20afa3244808952ddca337fbe5e8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/39ea2550643e4ae8bbf0ea1ee332e295, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/7fde3001ac4946d19d9ae2942f3df966, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f3cc01a5ee054c919b1104710e1cfa48, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/33fb4d85117a4a53bb254548ec971563, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/cacfcbcf84bd40ecbe434f0c4f167795, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/59bb0e43e8e44cd187affe8660f26d39, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/75cad00877754477b9ec10b91189348a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/0c6466af750744b79b99c2f0b9f660e3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/5d4ad24b27824a3d9cfe4c6c993a0d86, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/56f44be5594348bfa19cd1982c44212e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/a76448df610c4338b699b7d32561f282, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/452ad87502494fca8fd9622f82b6b0d7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/537769b1ca17466b82875dd73e1661bc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/82b71419786c4d9f9fd5bb3bbc58e8e2, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/16ef86fb7b7b4584a567240e1544aec5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/03be6587761443f38108c11e35ead0eb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/6ca9c4bc83b24e199b34c7d3d733abe9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/b50e769779184a4982a92cdd16e85df9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/22db1880d9f740bb9bbf5c45c47b4b9e] to archive 2024-11-11T07:24:11,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:24:11,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/daad491a0aae4780919ea70ec9330274 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/daad491a0aae4780919ea70ec9330274 2024-11-11T07:24:11,300 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/4cc79c4c1000423aaac1c82183c841a4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/4cc79c4c1000423aaac1c82183c841a4 2024-11-11T07:24:11,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/857a063be7d94fb3bab81d5f53c3948e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/857a063be7d94fb3bab81d5f53c3948e 2024-11-11T07:24:11,303 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/25fb20afa3244808952ddca337fbe5e8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/25fb20afa3244808952ddca337fbe5e8 2024-11-11T07:24:11,306 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/39ea2550643e4ae8bbf0ea1ee332e295 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/39ea2550643e4ae8bbf0ea1ee332e295 2024-11-11T07:24:11,307 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/7fde3001ac4946d19d9ae2942f3df966 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/7fde3001ac4946d19d9ae2942f3df966 2024-11-11T07:24:11,365 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f3cc01a5ee054c919b1104710e1cfa48 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f3cc01a5ee054c919b1104710e1cfa48 2024-11-11T07:24:11,377 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/33fb4d85117a4a53bb254548ec971563 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/33fb4d85117a4a53bb254548ec971563 2024-11-11T07:24:11,379 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/cacfcbcf84bd40ecbe434f0c4f167795 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/cacfcbcf84bd40ecbe434f0c4f167795 2024-11-11T07:24:11,380 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/59bb0e43e8e44cd187affe8660f26d39 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/59bb0e43e8e44cd187affe8660f26d39 2024-11-11T07:24:11,381 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/75cad00877754477b9ec10b91189348a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/75cad00877754477b9ec10b91189348a 2024-11-11T07:24:11,383 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/0c6466af750744b79b99c2f0b9f660e3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/0c6466af750744b79b99c2f0b9f660e3 2024-11-11T07:24:11,400 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/5d4ad24b27824a3d9cfe4c6c993a0d86 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/5d4ad24b27824a3d9cfe4c6c993a0d86 2024-11-11T07:24:11,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/56f44be5594348bfa19cd1982c44212e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/56f44be5594348bfa19cd1982c44212e 2024-11-11T07:24:11,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/a76448df610c4338b699b7d32561f282 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/a76448df610c4338b699b7d32561f282 2024-11-11T07:24:11,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/452ad87502494fca8fd9622f82b6b0d7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/452ad87502494fca8fd9622f82b6b0d7 2024-11-11T07:24:11,406 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/537769b1ca17466b82875dd73e1661bc to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/537769b1ca17466b82875dd73e1661bc 2024-11-11T07:24:11,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/82b71419786c4d9f9fd5bb3bbc58e8e2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/82b71419786c4d9f9fd5bb3bbc58e8e2 2024-11-11T07:24:11,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/16ef86fb7b7b4584a567240e1544aec5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/16ef86fb7b7b4584a567240e1544aec5 2024-11-11T07:24:11,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/03be6587761443f38108c11e35ead0eb to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/03be6587761443f38108c11e35ead0eb 2024-11-11T07:24:11,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/6ca9c4bc83b24e199b34c7d3d733abe9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/6ca9c4bc83b24e199b34c7d3d733abe9 2024-11-11T07:24:11,413 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/b50e769779184a4982a92cdd16e85df9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/b50e769779184a4982a92cdd16e85df9 2024-11-11T07:24:11,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/22db1880d9f740bb9bbf5c45c47b4b9e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/22db1880d9f740bb9bbf5c45c47b4b9e 2024-11-11T07:24:11,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8108726c802a4c3c999298bb3c1d7b4b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/337f2777d0d14b0cb9fe5205311086f9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/924d6f0ab1cf4c0697829ffd5c0b0f62, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8e895cb597b94eb7b08978e3383f6ee8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/aecd15498715459888320a2a0ad2613a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/a2585214e8564baea6a25739d9464793, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e90ed97e5caa4aa7ac85f8267d35b477, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c21c50b29a2b4b98a4aca7ceb7dcc75a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e62717e91ef64d14af61165bc9bb0591, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d5ea08dda0a94ae1ad8534ed3409c318, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e58c851e12f146309ecb7e068722b0fe, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f83ba60b95ba48a697ddeafd8b677df0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/71de7c05715a4ea3ac06dc2e195a9c25, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e7fd466a9e73476e83c3344877ac410c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d6c90ae61e0a43abb859a3d8f1ae7306, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7d6be24a13664e2d9cc8aa838d8a22ca, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/92b12d04e056422b8235234b1811d6d5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/5c765f8c68124537b5ab6caf707f7fc8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c5bebebc776849a3860687ae8514d28e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/681d73ae1efe4c7399d976a34d387fad, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/42e901681cc5418283fe6708468a39ca, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f1a30ec0c0e74741a4e87305f447cd3d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/34bfefec07ca462281c29b0b21a1369c] to archive 2024-11-11T07:24:11,486 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:24:11,488 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8108726c802a4c3c999298bb3c1d7b4b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8108726c802a4c3c999298bb3c1d7b4b 2024-11-11T07:24:11,490 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/337f2777d0d14b0cb9fe5205311086f9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/337f2777d0d14b0cb9fe5205311086f9 2024-11-11T07:24:11,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/924d6f0ab1cf4c0697829ffd5c0b0f62 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/924d6f0ab1cf4c0697829ffd5c0b0f62 2024-11-11T07:24:11,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8e895cb597b94eb7b08978e3383f6ee8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/8e895cb597b94eb7b08978e3383f6ee8 2024-11-11T07:24:11,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/aecd15498715459888320a2a0ad2613a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/aecd15498715459888320a2a0ad2613a 2024-11-11T07:24:11,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/a2585214e8564baea6a25739d9464793 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/a2585214e8564baea6a25739d9464793 2024-11-11T07:24:11,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e90ed97e5caa4aa7ac85f8267d35b477 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e90ed97e5caa4aa7ac85f8267d35b477 2024-11-11T07:24:11,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c21c50b29a2b4b98a4aca7ceb7dcc75a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c21c50b29a2b4b98a4aca7ceb7dcc75a 2024-11-11T07:24:11,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e62717e91ef64d14af61165bc9bb0591 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e62717e91ef64d14af61165bc9bb0591 2024-11-11T07:24:11,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d5ea08dda0a94ae1ad8534ed3409c318 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d5ea08dda0a94ae1ad8534ed3409c318 2024-11-11T07:24:11,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e58c851e12f146309ecb7e068722b0fe to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e58c851e12f146309ecb7e068722b0fe 2024-11-11T07:24:11,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f83ba60b95ba48a697ddeafd8b677df0 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f83ba60b95ba48a697ddeafd8b677df0 2024-11-11T07:24:11,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/71de7c05715a4ea3ac06dc2e195a9c25 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/71de7c05715a4ea3ac06dc2e195a9c25 2024-11-11T07:24:11,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e7fd466a9e73476e83c3344877ac410c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/e7fd466a9e73476e83c3344877ac410c 2024-11-11T07:24:11,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d6c90ae61e0a43abb859a3d8f1ae7306 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/d6c90ae61e0a43abb859a3d8f1ae7306 2024-11-11T07:24:11,632 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7d6be24a13664e2d9cc8aa838d8a22ca to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7d6be24a13664e2d9cc8aa838d8a22ca 2024-11-11T07:24:11,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/92b12d04e056422b8235234b1811d6d5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/92b12d04e056422b8235234b1811d6d5 2024-11-11T07:24:11,652 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/5c765f8c68124537b5ab6caf707f7fc8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/5c765f8c68124537b5ab6caf707f7fc8 2024-11-11T07:24:11,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c5bebebc776849a3860687ae8514d28e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/c5bebebc776849a3860687ae8514d28e 2024-11-11T07:24:11,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/681d73ae1efe4c7399d976a34d387fad to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/681d73ae1efe4c7399d976a34d387fad 2024-11-11T07:24:11,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/42e901681cc5418283fe6708468a39ca to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/42e901681cc5418283fe6708468a39ca 2024-11-11T07:24:11,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f1a30ec0c0e74741a4e87305f447cd3d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/f1a30ec0c0e74741a4e87305f447cd3d 2024-11-11T07:24:11,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f334e8b0fbdf:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/34bfefec07ca462281c29b0b21a1369c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/34bfefec07ca462281c29b0b21a1369c 2024-11-11T07:24:12,778 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-11T07:24:12,778 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:12,778 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:12,778 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:12,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:13,244 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-11T07:24:13,244 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:13,244 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:13,244 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:13,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 984 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2952 rows 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 984 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2951 rows 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 972 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2911 rows 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 981 2024-11-11T07:24:13,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2942 rows 2024-11-11T07:24:13,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 983 2024-11-11T07:24:13,245 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2948 rows 2024-11-11T07:24:13,245 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:24:13,245 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:24:13,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:13,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:13,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:13,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:24:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:24:13,271 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309853271"}]},"ts":"1731309853271"} 2024-11-11T07:24:13,273 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:24:13,273 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:24:13,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:24:13,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, UNASSIGN}] 2024-11-11T07:24:13,285 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, UNASSIGN 2024-11-11T07:24:13,286 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:13,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, UNASSIGN because future has completed 2024-11-11T07:24:13,293 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:24:13,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:24:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:24:13,451 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:13,451 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:24:13,451 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing b5411b408cdd3e1e96fba71c599080d2, disabling compactions & flushes 2024-11-11T07:24:13,451 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:13,451 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:13,451 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. after waiting 0 ms 2024-11-11T07:24:13,451 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:13,451 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing b5411b408cdd3e1e96fba71c599080d2 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-11T07:24:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:24:13,585 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111818a9e7d7a834c1daea0ecdf31adc9fe_b5411b408cdd3e1e96fba71c599080d2 is 50, key is test_row_1/A:col10/1731309852776/Put/seqid=0 2024-11-11T07:24:13,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742262_1438 (size=9914) 2024-11-11T07:24:13,648 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:13,764 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111818a9e7d7a834c1daea0ecdf31adc9fe_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111818a9e7d7a834c1daea0ecdf31adc9fe_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:13,765 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/6f4c7710237b45ff800cc571d2e6c285, store: [table=TestAcidGuarantees family=A region=b5411b408cdd3e1e96fba71c599080d2] 2024-11-11T07:24:13,766 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/6f4c7710237b45ff800cc571d2e6c285 is 175, key is test_row_1/A:col10/1731309852776/Put/seqid=0 2024-11-11T07:24:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:24:13,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742263_1439 (size=22561) 2024-11-11T07:24:13,968 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/6f4c7710237b45ff800cc571d2e6c285 2024-11-11T07:24:14,062 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/f9d41b46877b4876a04f2956d84b7f4e is 50, key is test_row_1/B:col10/1731309852776/Put/seqid=0 2024-11-11T07:24:14,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742264_1440 (size=9857) 2024-11-11T07:24:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:24:14,559 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/f9d41b46877b4876a04f2956d84b7f4e 2024-11-11T07:24:14,591 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/1104a15d64b74ab5862397bd303b7bb3 is 50, key is test_row_1/C:col10/1731309852776/Put/seqid=0 2024-11-11T07:24:14,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742265_1441 (size=9857) 2024-11-11T07:24:15,118 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/1104a15d64b74ab5862397bd303b7bb3 2024-11-11T07:24:15,122 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/A/6f4c7710237b45ff800cc571d2e6c285 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/6f4c7710237b45ff800cc571d2e6c285 2024-11-11T07:24:15,137 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/6f4c7710237b45ff800cc571d2e6c285, entries=100, sequenceid=350, filesize=22.0 K 2024-11-11T07:24:15,153 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/B/f9d41b46877b4876a04f2956d84b7f4e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f9d41b46877b4876a04f2956d84b7f4e 2024-11-11T07:24:15,165 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f9d41b46877b4876a04f2956d84b7f4e, entries=100, sequenceid=350, filesize=9.6 K 2024-11-11T07:24:15,172 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/.tmp/C/1104a15d64b74ab5862397bd303b7bb3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/1104a15d64b74ab5862397bd303b7bb3 2024-11-11T07:24:15,176 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/1104a15d64b74ab5862397bd303b7bb3, entries=100, sequenceid=350, filesize=9.6 K 2024-11-11T07:24:15,177 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b5411b408cdd3e1e96fba71c599080d2 in 1726ms, sequenceid=350, compaction requested=true 2024-11-11T07:24:15,344 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/recovered.edits/353.seqid, newMaxSeqId=353, maxSeqId=4 2024-11-11T07:24:15,345 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. 2024-11-11T07:24:15,345 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for b5411b408cdd3e1e96fba71c599080d2: Waiting for close lock at 1731309853451Running coprocessor pre-close hooks at 1731309853451Disabling compacts and flushes for region at 1731309853451Disabling writes for close at 1731309853451Obtaining lock to block concurrent updates at 1731309853452 (+1 ms)Preparing flush snapshotting stores in b5411b408cdd3e1e96fba71c599080d2 at 1731309853452Finished memstore snapshotting TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731309853452Flushing stores of TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2. at 1731309853452Flushing b5411b408cdd3e1e96fba71c599080d2/A: creating writer at 1731309853453 (+1 ms)Flushing mob file b5411b408cdd3e1e96fba71c599080d2/A: appending metadata at 1731309853584 (+131 ms)Flushing mob file b5411b408cdd3e1e96fba71c599080d2/A: closing flushed file at 1731309853585 (+1 ms)Flushing b5411b408cdd3e1e96fba71c599080d2/A: appending metadata at 1731309853765 (+180 ms)Flushing b5411b408cdd3e1e96fba71c599080d2/A: closing flushed file at 1731309853766 (+1 ms)Flushing b5411b408cdd3e1e96fba71c599080d2/B: creating writer at 1731309854031 (+265 ms)Flushing b5411b408cdd3e1e96fba71c599080d2/B: appending metadata at 1731309854061 (+30 ms)Flushing b5411b408cdd3e1e96fba71c599080d2/B: closing flushed file at 1731309854061Flushing b5411b408cdd3e1e96fba71c599080d2/C: creating writer at 1731309854568 (+507 ms)Flushing b5411b408cdd3e1e96fba71c599080d2/C: appending metadata at 1731309854591 (+23 ms)Flushing b5411b408cdd3e1e96fba71c599080d2/C: closing flushed file at 1731309854591Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d5cc142: reopening flushed file at 1731309855122 (+531 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@107d2b07: reopening flushed file at 1731309855137 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1649448e: reopening flushed file at 1731309855165 (+28 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b5411b408cdd3e1e96fba71c599080d2 in 1726ms, sequenceid=350, compaction requested=true at 1731309855177 (+12 ms)Writing region close event to WAL at 1731309855255 (+78 ms)Running coprocessor post-close hooks at 1731309855345 (+90 ms)Closed at 1731309855345 2024-11-11T07:24:15,348 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=b5411b408cdd3e1e96fba71c599080d2, regionState=CLOSED 2024-11-11T07:24:15,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:24:15,350 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T07:24:15,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure b5411b408cdd3e1e96fba71c599080d2, server=f334e8b0fbdf,39329,1731309728191 in 2.0580 sec 2024-11-11T07:24:15,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-11T07:24:15,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5411b408cdd3e1e96fba71c599080d2, UNASSIGN in 2.0790 sec 2024-11-11T07:24:15,360 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309855360"}]},"ts":"1731309855360"} 2024-11-11T07:24:15,362 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:24:15,362 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:24:15,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1170 sec 2024-11-11T07:24:15,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T07:24:15,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0840 sec 2024-11-11T07:24:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:24:15,425 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:15,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:24:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:15,428 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:15,429 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:24:15,465 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,471 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/recovered.edits] 2024-11-11T07:24:15,481 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/6f4c7710237b45ff800cc571d2e6c285 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/6f4c7710237b45ff800cc571d2e6c285 2024-11-11T07:24:15,483 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8e590a2fb39642d58b3a7ae08144747f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/8e590a2fb39642d58b3a7ae08144747f 2024-11-11T07:24:15,485 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/b2274637e0f94e95974659de28f8b5d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/A/b2274637e0f94e95974659de28f8b5d2 2024-11-11T07:24:15,489 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/962094d0ff8e44cd85d0b2461f9a94cf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/962094d0ff8e44cd85d0b2461f9a94cf 2024-11-11T07:24:15,490 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/9907cf43bd6b4f568521bb923f32c8b1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/9907cf43bd6b4f568521bb923f32c8b1 2024-11-11T07:24:15,491 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f9d41b46877b4876a04f2956d84b7f4e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/B/f9d41b46877b4876a04f2956d84b7f4e 2024-11-11T07:24:15,494 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/1104a15d64b74ab5862397bd303b7bb3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/1104a15d64b74ab5862397bd303b7bb3 2024-11-11T07:24:15,495 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7974338ae739441ba4d482a483463e3c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/7974338ae739441ba4d482a483463e3c 2024-11-11T07:24:15,496 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/faf0983e2d424cf8bb1145083f871863 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/C/faf0983e2d424cf8bb1145083f871863 2024-11-11T07:24:15,519 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/recovered.edits/353.seqid to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2/recovered.edits/353.seqid 2024-11-11T07:24:15,520 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,520 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:24:15,521 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:24:15,540 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T07:24:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:24:15,585 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111006564c7940e4d94871471a16c0a8e0e_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111006564c7940e4d94871471a16c0a8e0e_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,587 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110eec6270d2904ec197181099eec13c1a_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110eec6270d2904ec197181099eec13c1a_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,588 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b3cc9fc6e18493fb54fadc5246b5286_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112b3cc9fc6e18493fb54fadc5246b5286_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,589 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111314c49fe3a7b4fa7ac860c4ea5aa1593_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111314c49fe3a7b4fa7ac860c4ea5aa1593_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,591 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113a13ad71a9fb4d61abd6b33bb9d5017b_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113a13ad71a9fb4d61abd6b33bb9d5017b_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,592 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113fbb7d723b324790809c377318fa965d_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113fbb7d723b324790809c377318fa965d_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,593 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114980ae1b791d4950af187ed19a5c1c43_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114980ae1b791d4950af187ed19a5c1c43_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,594 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e64ad5d029944188e9071b64dd381c8_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e64ad5d029944188e9071b64dd381c8_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,596 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163234171d7f841748587cebfd28de11f_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163234171d7f841748587cebfd28de11f_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,601 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ca76703a0e84d80a3027e78ff32332f_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ca76703a0e84d80a3027e78ff32332f_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,602 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111818a9e7d7a834c1daea0ecdf31adc9fe_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111818a9e7d7a834c1daea0ecdf31adc9fe_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,603 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118869e721d9f541d9b21d52cccec9152d_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118869e721d9f541d9b21d52cccec9152d_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,605 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119826b6a5a34a42eeb8488790a05fb78c_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119826b6a5a34a42eeb8488790a05fb78c_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,606 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae15cc245cdc4b0e9895888ad3d077d3_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae15cc245cdc4b0e9895888ad3d077d3_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b1644b74812e4f9a892229353c52a36b_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b1644b74812e4f9a892229353c52a36b_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,608 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d2e3b54a0edb4d0da392069c4077b680_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d2e3b54a0edb4d0da392069c4077b680_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,610 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df470b18f6dc46349c6e9c6cfef78ec8_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df470b18f6dc46349c6e9c6cfef78ec8_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,611 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e31e41373c5543039484c877812f6fce_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e31e41373c5543039484c877812f6fce_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,632 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e4cd55f573554a10a08fcba7729d565c_b5411b408cdd3e1e96fba71c599080d2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e4cd55f573554a10a08fcba7729d565c_b5411b408cdd3e1e96fba71c599080d2 2024-11-11T07:24:15,632 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:24:15,644 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:15,648 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:24:15,652 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:24:15,656 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:15,656 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:24:15,657 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731309855656"}]},"ts":"9223372036854775807"} 2024-11-11T07:24:15,664 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:24:15,664 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b5411b408cdd3e1e96fba71c599080d2, NAME => 'TestAcidGuarantees,,1731309819914.b5411b408cdd3e1e96fba71c599080d2.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:24:15,664 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:24:15,664 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731309855664"}]},"ts":"9223372036854775807"} 2024-11-11T07:24:15,672 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:24:15,674 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:15,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 248 msec 2024-11-11T07:24:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:24:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:24:15,752 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:15,801 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 240), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1453 (was 492) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 12), AvailableMemoryMB=7902 (was 7964) 2024-11-11T07:24:15,946 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1453, ProcessCount=11, AvailableMemoryMB=7900 2024-11-11T07:24:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:24:15,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:24:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:15,951 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:24:15,951 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:15,952 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:24:15,952 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-11T07:24:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T07:24:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T07:24:16,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742266_1442 (size=959) 2024-11-11T07:24:16,215 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:24:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T07:24:16,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742267_1443 (size=53) 2024-11-11T07:24:16,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:24:16,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 45154c5da223c60a66ac44ba1cc3400c, disabling compactions & flushes 2024-11-11T07:24:16,387 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:16,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:16,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. after waiting 0 ms 2024-11-11T07:24:16,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:16,387 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:16,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45154c5da223c60a66ac44ba1cc3400c: Waiting for close lock at 1731309856387Disabling compacts and flushes for region at 1731309856387Disabling writes for close at 1731309856387Writing region close event to WAL at 1731309856387Closed at 1731309856387 2024-11-11T07:24:16,388 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:24:16,389 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731309856389"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731309856389"}]},"ts":"1731309856389"} 2024-11-11T07:24:16,391 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:24:16,392 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:24:16,392 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309856392"}]},"ts":"1731309856392"} 2024-11-11T07:24:16,396 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:24:16,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, ASSIGN}] 2024-11-11T07:24:16,397 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, ASSIGN 2024-11-11T07:24:16,399 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, ASSIGN; state=OFFLINE, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=false 2024-11-11T07:24:16,550 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=45154c5da223c60a66ac44ba1cc3400c, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:16,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, ASSIGN because future has completed 2024-11-11T07:24:16,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:24:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T07:24:16,748 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:16,748 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:24:16,749 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,749 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:24:16,750 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,750 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,773 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,842 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45154c5da223c60a66ac44ba1cc3400c columnFamilyName A 2024-11-11T07:24:16,842 DEBUG [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:16,856 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] regionserver.HStore(327): Store=45154c5da223c60a66ac44ba1cc3400c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:16,856 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,864 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45154c5da223c60a66ac44ba1cc3400c columnFamilyName B 2024-11-11T07:24:16,864 DEBUG [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:16,889 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] regionserver.HStore(327): Store=45154c5da223c60a66ac44ba1cc3400c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:16,889 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,894 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45154c5da223c60a66ac44ba1cc3400c columnFamilyName C 2024-11-11T07:24:16,894 DEBUG [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:16,894 INFO [StoreOpener-45154c5da223c60a66ac44ba1cc3400c-1 {}] regionserver.HStore(327): Store=45154c5da223c60a66ac44ba1cc3400c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:16,894 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,895 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,895 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,896 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,896 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,896 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:24:16,906 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,937 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:24:16,939 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 45154c5da223c60a66ac44ba1cc3400c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69113241, jitterRate=0.029867544770240784}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:24:16,939 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:16,940 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 45154c5da223c60a66ac44ba1cc3400c: Running coprocessor pre-open hook at 1731309856750Writing region info on filesystem at 1731309856750Initializing all the Stores at 1731309856750Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309856751 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309856773 (+22 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309856773Cleaning up temporary data from old regions at 1731309856896 (+123 ms)Running coprocessor post-open hooks at 1731309856939 (+43 ms)Region opened successfully at 1731309856940 (+1 ms) 2024-11-11T07:24:16,941 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., pid=116, masterSystemTime=1731309856709 2024-11-11T07:24:16,943 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=45154c5da223c60a66ac44ba1cc3400c, regionState=OPEN, openSeqNum=2, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:16,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:24:16,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T07:24:16,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 in 390 msec 2024-11-11T07:24:16,949 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:16,949 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:16,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-11T07:24:16,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, ASSIGN in 553 msec 2024-11-11T07:24:16,952 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:24:16,953 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309856952"}]},"ts":"1731309856952"} 2024-11-11T07:24:16,954 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:24:16,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:24:16,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.0090 sec 2024-11-11T07:24:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T07:24:17,102 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:17,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005a7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,130 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c8a5b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,133 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7292ee9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,134 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,137 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:24:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f334e8b0fbdf,39865,1731309727465 2024-11-11T07:24:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51874cca 2024-11-11T07:24:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:24:17,140 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:24:17,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,169 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5b2451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134f601a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,221 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646a9124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,224 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e472f3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,283 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed0bd1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,286 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301f6b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,366 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbc7f35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,368 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176eb3e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58bbb759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384b2dfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d367c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8d37c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b369e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,458 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fdd9228, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,461 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747d71cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,463 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ad26e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,468 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17fb0ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,470 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b4d685b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:17,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:17,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e60a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:17,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af4a541, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:17,502 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,503 DEBUG [Thread-2005 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,503 DEBUG [Thread-2005 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,504 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,505 DEBUG [Thread-1992 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,505 DEBUG [Thread-1992 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,508 DEBUG [Thread-1992 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,519 DEBUG [Thread-1996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,540 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:17,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:24:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:17,545 DEBUG [Thread-2009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,547 DEBUG [Thread-2009 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,547 DEBUG [Thread-2009 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,547 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,548 DEBUG [Thread-2003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,548 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T07:24:17,550 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:17,551 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:17,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,567 DEBUG [Thread-2009 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:24:17,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:24:17,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:17,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:17,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:17,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:17,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:17,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:17,741 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,743 DEBUG [Thread-2000 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,743 DEBUG [Thread-2000 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,749 DEBUG [Thread-2000 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,776 DEBUG [Thread-2011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,778 DEBUG [Thread-2011 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,778 DEBUG [Thread-2011 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,785 DEBUG [Thread-2011 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,793 DEBUG [Thread-1998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,803 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:17,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2] 2024-11-11T07:24:17,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:17,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,012 INFO [AsyncFSWAL-0-hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData-prefix:f334e8b0fbdf,39865,1731309727465 {}] wal.AbstractFSWAL(1368): Slow sync cost: 141 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40127,DS-74a6e686-2d71-4357-ae40-76976cd3e84c,DISK]] 2024-11-11T07:24:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:24:18,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9208737c389d4ede9cc4927f989e47cf is 50, key is test_row_0/A:col10/1731309857538/Put/seqid=0 2024-11-11T07:24:18,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309918055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309918061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309918103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309918105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309918105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742268_1444 (size=12001) 2024-11-11T07:24:18,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:18,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309918213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309918220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309918222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309918232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309918232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:24:18,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:18,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:18,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9208737c389d4ede9cc4927f989e47cf 2024-11-11T07:24:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309918522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309918522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309918522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309918522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309918522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:18,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/583a9a9ed90b46a09288b1ae12ceb258 is 50, key is test_row_0/B:col10/1731309857538/Put/seqid=0 2024-11-11T07:24:18,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742269_1445 (size=12001) 2024-11-11T07:24:18,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/583a9a9ed90b46a09288b1ae12ceb258 2024-11-11T07:24:18,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:24:18,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/0bc115a497494b72ae0ea633982c2fdd is 50, key is test_row_0/C:col10/1731309857538/Put/seqid=0 2024-11-11T07:24:18,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:18,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:19,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742270_1446 (size=12001) 2024-11-11T07:24:19,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/0bc115a497494b72ae0ea633982c2fdd 2024-11-11T07:24:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309918901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:19,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:19,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309918902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309918902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:19,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309918909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309918902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9208737c389d4ede9cc4927f989e47cf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9208737c389d4ede9cc4927f989e47cf 2024-11-11T07:24:19,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9208737c389d4ede9cc4927f989e47cf, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:24:19,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/583a9a9ed90b46a09288b1ae12ceb258 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/583a9a9ed90b46a09288b1ae12ceb258 2024-11-11T07:24:19,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:19,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:19,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/583a9a9ed90b46a09288b1ae12ceb258, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:24:19,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/0bc115a497494b72ae0ea633982c2fdd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/0bc115a497494b72ae0ea633982c2fdd 2024-11-11T07:24:19,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/0bc115a497494b72ae0ea633982c2fdd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:24:19,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 45154c5da223c60a66ac44ba1cc3400c in 1629ms, sequenceid=12, compaction requested=false 2024-11-11T07:24:19,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:19,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T07:24:19,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:19,350 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:24:19,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b5c3bdd4d1941849386a90aea39b66d is 50, key is test_row_0/A:col10/1731309858102/Put/seqid=0 2024-11-11T07:24:19,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742271_1447 (size=12001) 2024-11-11T07:24:19,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:24:19,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309919728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309919729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:19,944 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b5c3bdd4d1941849386a90aea39b66d 2024-11-11T07:24:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/0cbd102f38d9431a9900eaf0c988ea16 is 50, key is test_row_0/B:col10/1731309858102/Put/seqid=0 2024-11-11T07:24:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309919892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309919921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309919921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742272_1448 (size=12001) 2024-11-11T07:24:20,083 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/0cbd102f38d9431a9900eaf0c988ea16 2024-11-11T07:24:20,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309920032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/99750b9528014c12a0c17b32a6bffaee is 50, key is test_row_0/C:col10/1731309858102/Put/seqid=0 2024-11-11T07:24:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309920045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309920153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309920154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309920158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742273_1449 (size=12001) 2024-11-11T07:24:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309920304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309920372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309920463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309920463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309920471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:20,676 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/99750b9528014c12a0c17b32a6bffaee 2024-11-11T07:24:20,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b5c3bdd4d1941849386a90aea39b66d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b5c3bdd4d1941849386a90aea39b66d 2024-11-11T07:24:20,685 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b5c3bdd4d1941849386a90aea39b66d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T07:24:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/0cbd102f38d9431a9900eaf0c988ea16 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/0cbd102f38d9431a9900eaf0c988ea16 2024-11-11T07:24:20,690 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/0cbd102f38d9431a9900eaf0c988ea16, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T07:24:20,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/99750b9528014c12a0c17b32a6bffaee as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/99750b9528014c12a0c17b32a6bffaee 2024-11-11T07:24:20,695 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/99750b9528014c12a0c17b32a6bffaee, entries=150, sequenceid=37, filesize=11.7 K 2024-11-11T07:24:20,697 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 45154c5da223c60a66ac44ba1cc3400c in 1347ms, sequenceid=37, compaction requested=false 2024-11-11T07:24:20,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:20,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:20,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T07:24:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T07:24:20,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T07:24:20,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1480 sec 2024-11-11T07:24:20,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.1730 sec 2024-11-11T07:24:20,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:20,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:24:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:20,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/a89d0d1717fa4c5db38b60e43fd40308 is 50, key is test_row_0/A:col10/1731309859919/Put/seqid=0 2024-11-11T07:24:20,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:24:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742274_1450 (size=14341) 2024-11-11T07:24:20,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/a89d0d1717fa4c5db38b60e43fd40308 2024-11-11T07:24:21,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/eb1c0419a61141caa8eae52ba0ac0265 is 50, key is test_row_0/B:col10/1731309859919/Put/seqid=0 2024-11-11T07:24:21,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742275_1451 (size=12001) 2024-11-11T07:24:21,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/eb1c0419a61141caa8eae52ba0ac0265 2024-11-11T07:24:21,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/03775f2f6ee04c2b8f1787346fb5ecf5 is 50, key is test_row_0/C:col10/1731309859919/Put/seqid=0 2024-11-11T07:24:21,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742276_1452 (size=12001) 2024-11-11T07:24:21,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/03775f2f6ee04c2b8f1787346fb5ecf5 2024-11-11T07:24:21,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/a89d0d1717fa4c5db38b60e43fd40308 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a89d0d1717fa4c5db38b60e43fd40308 2024-11-11T07:24:21,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a89d0d1717fa4c5db38b60e43fd40308, entries=200, sequenceid=49, filesize=14.0 K 2024-11-11T07:24:21,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/eb1c0419a61141caa8eae52ba0ac0265 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/eb1c0419a61141caa8eae52ba0ac0265 2024-11-11T07:24:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309921441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:21,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/eb1c0419a61141caa8eae52ba0ac0265, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T07:24:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309921528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:21,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309921536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:21,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309921542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309921553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:21,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/03775f2f6ee04c2b8f1787346fb5ecf5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/03775f2f6ee04c2b8f1787346fb5ecf5 2024-11-11T07:24:21,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/03775f2f6ee04c2b8f1787346fb5ecf5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T07:24:21,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 45154c5da223c60a66ac44ba1cc3400c in 766ms, sequenceid=49, compaction requested=true 2024-11-11T07:24:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:21,601 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:21,601 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:21,602 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:21,602 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/A is initiating minor compaction (all files) 2024-11-11T07:24:21,602 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/A in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:21,602 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9208737c389d4ede9cc4927f989e47cf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b5c3bdd4d1941849386a90aea39b66d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a89d0d1717fa4c5db38b60e43fd40308] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=37.4 K 2024-11-11T07:24:21,604 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9208737c389d4ede9cc4927f989e47cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731309857530 2024-11-11T07:24:21,604 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:21,604 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/B is initiating minor compaction (all files) 2024-11-11T07:24:21,604 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/B in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:21,604 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/583a9a9ed90b46a09288b1ae12ceb258, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/0cbd102f38d9431a9900eaf0c988ea16, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/eb1c0419a61141caa8eae52ba0ac0265] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=35.2 K 2024-11-11T07:24:21,604 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 583a9a9ed90b46a09288b1ae12ceb258, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731309857530 2024-11-11T07:24:21,604 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5c3bdd4d1941849386a90aea39b66d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731309858053 2024-11-11T07:24:21,605 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a89d0d1717fa4c5db38b60e43fd40308, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731309859691 2024-11-11T07:24:21,605 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbd102f38d9431a9900eaf0c988ea16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731309858053 2024-11-11T07:24:21,605 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting eb1c0419a61141caa8eae52ba0ac0265, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731309859715 2024-11-11T07:24:21,622 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#A#compaction#380 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:21,622 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/093a475914e947da9adc60ab1d21077f is 50, key is test_row_0/A:col10/1731309859919/Put/seqid=0 2024-11-11T07:24:21,674 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:21,675 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/545d05495a7e4578b0b5b68f95b92e85 is 50, key is test_row_0/B:col10/1731309859919/Put/seqid=0 2024-11-11T07:24:21,762 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:21,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T07:24:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:24:21,865 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:21,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T07:24:21,868 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:21,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:21,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T07:24:21,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309921768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:21,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309921777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:21,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:21,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742277_1453 (size=12104) 2024-11-11T07:24:21,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/3f92bbfe1d75480e91cd1b7c4538687c is 50, key is test_row_0/A:col10/1731309861761/Put/seqid=0 2024-11-11T07:24:21,976 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/093a475914e947da9adc60ab1d21077f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/093a475914e947da9adc60ab1d21077f 2024-11-11T07:24:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T07:24:21,982 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/A of 45154c5da223c60a66ac44ba1cc3400c into 093a475914e947da9adc60ab1d21077f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:21,982 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:21,982 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/A, priority=13, startTime=1731309861601; duration=0sec 2024-11-11T07:24:21,982 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:21,982 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:A 2024-11-11T07:24:21,982 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:21,983 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:21,983 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/C is initiating minor compaction (all files) 2024-11-11T07:24:21,983 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/C in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:21,983 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/0bc115a497494b72ae0ea633982c2fdd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/99750b9528014c12a0c17b32a6bffaee, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/03775f2f6ee04c2b8f1787346fb5ecf5] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=35.2 K 2024-11-11T07:24:21,983 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc115a497494b72ae0ea633982c2fdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731309857530 2024-11-11T07:24:21,984 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99750b9528014c12a0c17b32a6bffaee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731309858053 2024-11-11T07:24:21,984 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03775f2f6ee04c2b8f1787346fb5ecf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731309859715 2024-11-11T07:24:22,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309921873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309921876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309921880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742278_1454 (size=12104) 2024-11-11T07:24:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309921992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309921992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:22,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:22,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,022 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:22,023 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/048518a36e11418b858aa615630a230d is 50, key is test_row_0/C:col10/1731309859919/Put/seqid=0 2024-11-11T07:24:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,026 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/545d05495a7e4578b0b5b68f95b92e85 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/545d05495a7e4578b0b5b68f95b92e85 2024-11-11T07:24:22,034 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/B of 45154c5da223c60a66ac44ba1cc3400c into 545d05495a7e4578b0b5b68f95b92e85(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:22,034 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:22,034 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/B, priority=13, startTime=1731309861601; duration=0sec 2024-11-11T07:24:22,034 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:22,034 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:B 2024-11-11T07:24:22,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309922122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309922122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309922123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T07:24:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742279_1455 (size=12001) 2024-11-11T07:24:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/3f92bbfe1d75480e91cd1b7c4538687c 2024-11-11T07:24:22,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309922222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742280_1456 (size=12104) 2024-11-11T07:24:22,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309922244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/089a82fc1baf4726995b46faf6e64ba4 is 50, key is test_row_0/B:col10/1731309861761/Put/seqid=0 2024-11-11T07:24:22,322 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/048518a36e11418b858aa615630a230d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/048518a36e11418b858aa615630a230d 2024-11-11T07:24:22,328 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/C of 45154c5da223c60a66ac44ba1cc3400c into 048518a36e11418b858aa615630a230d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:22,328 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:22,328 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/C, priority=13, startTime=1731309861601; duration=0sec 2024-11-11T07:24:22,328 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:22,328 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:C 2024-11-11T07:24:22,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:22,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:22,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309922372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309922381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309922404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742281_1457 (size=12001) 2024-11-11T07:24:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T07:24:22,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309922601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309922621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309922752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309922752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309922752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/089a82fc1baf4726995b46faf6e64ba4 2024-11-11T07:24:22,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/b0089256976749d0b4b0eaae3511558a is 50, key is test_row_0/C:col10/1731309861761/Put/seqid=0 2024-11-11T07:24:22,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:22,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T07:24:23,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742282_1458 (size=12001) 2024-11-11T07:24:23,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:23,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309923157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309923157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:23,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:23,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:23,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309923331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:23,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:23,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309923345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:23,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:23,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:23,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309923358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:23,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:23,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/b0089256976749d0b4b0eaae3511558a 2024-11-11T07:24:23,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/3f92bbfe1d75480e91cd1b7c4538687c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3f92bbfe1d75480e91cd1b7c4538687c 2024-11-11T07:24:23,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3f92bbfe1d75480e91cd1b7c4538687c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T07:24:23,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/089a82fc1baf4726995b46faf6e64ba4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/089a82fc1baf4726995b46faf6e64ba4 2024-11-11T07:24:23,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/089a82fc1baf4726995b46faf6e64ba4, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T07:24:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/b0089256976749d0b4b0eaae3511558a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b0089256976749d0b4b0eaae3511558a 2024-11-11T07:24:23,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b0089256976749d0b4b0eaae3511558a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T07:24:23,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 45154c5da223c60a66ac44ba1cc3400c in 1708ms, sequenceid=77, compaction requested=false 2024-11-11T07:24:23,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:23,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T07:24:23,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:23,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T07:24:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/bec8ad8e106345869ba36513f3326e84 is 50, key is test_row_0/A:col10/1731309861874/Put/seqid=0 2024-11-11T07:24:23,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742283_1459 (size=9657) 2024-11-11T07:24:23,689 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/bec8ad8e106345869ba36513f3326e84 2024-11-11T07:24:23,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/6e13de21e17744699a6506653d7d7e71 is 50, key is test_row_0/B:col10/1731309861874/Put/seqid=0 2024-11-11T07:24:23,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742284_1460 (size=9657) 2024-11-11T07:24:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T07:24:24,206 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/6e13de21e17744699a6506653d7d7e71 2024-11-11T07:24:24,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/b3e2eeeb68a14f97bd5ae29d45f0af34 is 50, key is test_row_0/C:col10/1731309861874/Put/seqid=0 2024-11-11T07:24:24,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:24,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742285_1461 (size=9657) 2024-11-11T07:24:24,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309924392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309924392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309924431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,436 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2557 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309924442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309924442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,448 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2574 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,448 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2572 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309924501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309924512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,671 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/b3e2eeeb68a14f97bd5ae29d45f0af34 2024-11-11T07:24:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/bec8ad8e106345869ba36513f3326e84 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bec8ad8e106345869ba36513f3326e84 2024-11-11T07:24:24,682 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bec8ad8e106345869ba36513f3326e84, entries=100, sequenceid=88, filesize=9.4 K 2024-11-11T07:24:24,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/6e13de21e17744699a6506653d7d7e71 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/6e13de21e17744699a6506653d7d7e71 2024-11-11T07:24:24,690 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/6e13de21e17744699a6506653d7d7e71, entries=100, sequenceid=88, filesize=9.4 K 2024-11-11T07:24:24,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/b3e2eeeb68a14f97bd5ae29d45f0af34 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b3e2eeeb68a14f97bd5ae29d45f0af34 2024-11-11T07:24:24,695 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b3e2eeeb68a14f97bd5ae29d45f0af34, entries=100, sequenceid=88, filesize=9.4 K 2024-11-11T07:24:24,697 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 45154c5da223c60a66ac44ba1cc3400c in 1153ms, sequenceid=88, compaction requested=true 2024-11-11T07:24:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T07:24:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T07:24:24,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T07:24:24,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8300 sec 2024-11-11T07:24:24,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.8360 sec 2024-11-11T07:24:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:24,717 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:24,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-11T07:24:24,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/bb17a0729a264ceabf850b0655ec9a2f is 50, key is test_row_0/A:col10/1731309864716/Put/seqid=0 2024-11-11T07:24:24,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742286_1462 (size=14341) 2024-11-11T07:24:24,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/bb17a0729a264ceabf850b0655ec9a2f 2024-11-11T07:24:24,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309924741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/cbbb7d8de70149e1a050e2872e0a9ac0 is 50, key is test_row_0/B:col10/1731309864716/Put/seqid=0 2024-11-11T07:24:24,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309924744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742287_1463 (size=12001) 2024-11-11T07:24:24,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/cbbb7d8de70149e1a050e2872e0a9ac0 2024-11-11T07:24:24,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/afc69ad7db1c43f58fd1a9b464596c99 is 50, key is test_row_0/C:col10/1731309864716/Put/seqid=0 2024-11-11T07:24:24,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742288_1464 (size=12001) 2024-11-11T07:24:24,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/afc69ad7db1c43f58fd1a9b464596c99 2024-11-11T07:24:24,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/bb17a0729a264ceabf850b0655ec9a2f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bb17a0729a264ceabf850b0655ec9a2f 2024-11-11T07:24:24,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bb17a0729a264ceabf850b0655ec9a2f, entries=200, sequenceid=117, filesize=14.0 K 2024-11-11T07:24:24,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/cbbb7d8de70149e1a050e2872e0a9ac0 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/cbbb7d8de70149e1a050e2872e0a9ac0 2024-11-11T07:24:24,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/cbbb7d8de70149e1a050e2872e0a9ac0, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T07:24:24,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/afc69ad7db1c43f58fd1a9b464596c99 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/afc69ad7db1c43f58fd1a9b464596c99 2024-11-11T07:24:24,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/afc69ad7db1c43f58fd1a9b464596c99, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T07:24:24,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 45154c5da223c60a66ac44ba1cc3400c in 92ms, sequenceid=117, compaction requested=true 2024-11-11T07:24:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:24,810 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:24,810 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:24,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:24,811 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48103 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:24,811 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:24,811 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/B is initiating minor compaction (all files) 2024-11-11T07:24:24,811 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/A is initiating minor compaction (all files) 2024-11-11T07:24:24,811 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/B in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:24,811 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/A in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:24,811 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/093a475914e947da9adc60ab1d21077f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3f92bbfe1d75480e91cd1b7c4538687c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bec8ad8e106345869ba36513f3326e84, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bb17a0729a264ceabf850b0655ec9a2f] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=47.0 K 2024-11-11T07:24:24,811 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/545d05495a7e4578b0b5b68f95b92e85, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/089a82fc1baf4726995b46faf6e64ba4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/6e13de21e17744699a6506653d7d7e71, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/cbbb7d8de70149e1a050e2872e0a9ac0] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=44.7 K 2024-11-11T07:24:24,812 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 545d05495a7e4578b0b5b68f95b92e85, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731309859715 2024-11-11T07:24:24,812 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 093a475914e947da9adc60ab1d21077f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731309859715 2024-11-11T07:24:24,812 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 089a82fc1baf4726995b46faf6e64ba4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731309861525 2024-11-11T07:24:24,813 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f92bbfe1d75480e91cd1b7c4538687c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731309861525 2024-11-11T07:24:24,813 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e13de21e17744699a6506653d7d7e71, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731309861874 2024-11-11T07:24:24,813 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec8ad8e106345869ba36513f3326e84, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731309861874 2024-11-11T07:24:24,813 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbb7d8de70149e1a050e2872e0a9ac0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731309864376 2024-11-11T07:24:24,813 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb17a0729a264ceabf850b0655ec9a2f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731309864376 2024-11-11T07:24:24,825 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:24,825 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9aff86e17c7d497fb14e3b4b4931aff9 is 50, key is test_row_0/A:col10/1731309864716/Put/seqid=0 2024-11-11T07:24:24,828 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:24,828 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/919a1342b41241e09d43c7a3044b7465 is 50, key is test_row_0/B:col10/1731309864716/Put/seqid=0 2024-11-11T07:24:24,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742290_1466 (size=12241) 2024-11-11T07:24:24,868 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/919a1342b41241e09d43c7a3044b7465 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/919a1342b41241e09d43c7a3044b7465 2024-11-11T07:24:24,876 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/B of 45154c5da223c60a66ac44ba1cc3400c into 919a1342b41241e09d43c7a3044b7465(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:24,876 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:24,876 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/B, priority=12, startTime=1731309864810; duration=0sec 2024-11-11T07:24:24,876 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:24,876 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:B 2024-11-11T07:24:24,876 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:24,876 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:24,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:24:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:24,878 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:24,878 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/C is initiating minor compaction (all files) 2024-11-11T07:24:24,878 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/C in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:24,878 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/048518a36e11418b858aa615630a230d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b0089256976749d0b4b0eaae3511558a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b3e2eeeb68a14f97bd5ae29d45f0af34, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/afc69ad7db1c43f58fd1a9b464596c99] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=44.7 K 2024-11-11T07:24:24,879 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 048518a36e11418b858aa615630a230d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731309859715 2024-11-11T07:24:24,879 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b0089256976749d0b4b0eaae3511558a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731309861525 2024-11-11T07:24:24,879 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e2eeeb68a14f97bd5ae29d45f0af34, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731309861874 2024-11-11T07:24:24,880 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting afc69ad7db1c43f58fd1a9b464596c99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731309864376 2024-11-11T07:24:24,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742289_1465 (size=12241) 2024-11-11T07:24:24,892 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#C#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:24,892 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/2a0a0c537c9543149707d5f850abf4a6 is 50, key is test_row_0/C:col10/1731309864716/Put/seqid=0 2024-11-11T07:24:24,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/2235f4c82f2b43f3a0858c98f6dd3d60 is 50, key is test_row_0/A:col10/1731309864875/Put/seqid=0 2024-11-11T07:24:24,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742291_1467 (size=12241) 2024-11-11T07:24:24,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742292_1468 (size=16781) 2024-11-11T07:24:24,911 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/2a0a0c537c9543149707d5f850abf4a6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/2a0a0c537c9543149707d5f850abf4a6 2024-11-11T07:24:24,914 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/C of 45154c5da223c60a66ac44ba1cc3400c into 2a0a0c537c9543149707d5f850abf4a6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:24,914 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:24,914 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/C, priority=12, startTime=1731309864810; duration=0sec 2024-11-11T07:24:24,915 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:24,915 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:C 2024-11-11T07:24:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309924967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309924970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:25,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309925081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:25,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309925091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:25,287 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9aff86e17c7d497fb14e3b4b4931aff9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9aff86e17c7d497fb14e3b4b4931aff9 2024-11-11T07:24:25,290 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/A of 45154c5da223c60a66ac44ba1cc3400c into 9aff86e17c7d497fb14e3b4b4931aff9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:25,290 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:25,290 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/A, priority=12, startTime=1731309864810; duration=0sec 2024-11-11T07:24:25,290 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:25,290 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:A 2024-11-11T07:24:25,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309925302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:25,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309925302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:25,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/2235f4c82f2b43f3a0858c98f6dd3d60 2024-11-11T07:24:25,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/c4a86409011341639f69d4f84ad9a06c is 50, key is test_row_0/B:col10/1731309864875/Put/seqid=0 2024-11-11T07:24:25,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742293_1469 (size=12051) 2024-11-11T07:24:25,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309925611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:25,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309925611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:25,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/c4a86409011341639f69d4f84ad9a06c 2024-11-11T07:24:25,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/3960150445fd4dde94c3316ec91258eb is 50, key is test_row_0/C:col10/1731309864875/Put/seqid=0 2024-11-11T07:24:25,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742294_1470 (size=12051) 2024-11-11T07:24:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T07:24:26,042 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:26,042 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T07:24:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T07:24:26,043 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:26,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:26,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:26,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/3960150445fd4dde94c3316ec91258eb 2024-11-11T07:24:26,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/2235f4c82f2b43f3a0858c98f6dd3d60 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/2235f4c82f2b43f3a0858c98f6dd3d60 2024-11-11T07:24:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/2235f4c82f2b43f3a0858c98f6dd3d60, entries=250, sequenceid=130, filesize=16.4 K 2024-11-11T07:24:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309926131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309926131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/c4a86409011341639f69d4f84ad9a06c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c4a86409011341639f69d4f84ad9a06c 2024-11-11T07:24:26,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c4a86409011341639f69d4f84ad9a06c, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T07:24:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/3960150445fd4dde94c3316ec91258eb as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3960150445fd4dde94c3316ec91258eb 2024-11-11T07:24:26,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3960150445fd4dde94c3316ec91258eb, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T07:24:26,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 45154c5da223c60a66ac44ba1cc3400c in 1271ms, sequenceid=130, compaction requested=false 2024-11-11T07:24:26,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T07:24:26,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T07:24:26,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:26,197 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:24:26,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b3d055799ae498db7b41441ebef5ce9 is 50, key is test_row_0/A:col10/1731309864948/Put/seqid=0 2024-11-11T07:24:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742295_1471 (size=12151) 2024-11-11T07:24:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T07:24:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:26,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309926487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309926489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309926490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309926591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,607 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b3d055799ae498db7b41441ebef5ce9 2024-11-11T07:24:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309926601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309926601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/3ac63774f9614eaabb7803d7baff234c is 50, key is test_row_0/B:col10/1731309864948/Put/seqid=0 2024-11-11T07:24:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742296_1472 (size=12151) 2024-11-11T07:24:26,625 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/3ac63774f9614eaabb7803d7baff234c 2024-11-11T07:24:26,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/bc80bf777d5a4b319033b25fc8d04e1e is 50, key is test_row_0/C:col10/1731309864948/Put/seqid=0 2024-11-11T07:24:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742297_1473 (size=12151) 2024-11-11T07:24:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T07:24:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309926802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309926822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309926822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,042 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/bc80bf777d5a4b319033b25fc8d04e1e 2024-11-11T07:24:27,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b3d055799ae498db7b41441ebef5ce9 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b3d055799ae498db7b41441ebef5ce9 2024-11-11T07:24:27,050 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b3d055799ae498db7b41441ebef5ce9, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:24:27,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/3ac63774f9614eaabb7803d7baff234c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/3ac63774f9614eaabb7803d7baff234c 2024-11-11T07:24:27,054 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/3ac63774f9614eaabb7803d7baff234c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:24:27,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/bc80bf777d5a4b319033b25fc8d04e1e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/bc80bf777d5a4b319033b25fc8d04e1e 2024-11-11T07:24:27,057 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/bc80bf777d5a4b319033b25fc8d04e1e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T07:24:27,058 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 45154c5da223c60a66ac44ba1cc3400c in 862ms, sequenceid=156, compaction requested=true 2024-11-11T07:24:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T07:24:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T07:24:27,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T07:24:27,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-11T07:24:27,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0200 sec 2024-11-11T07:24:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:27,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:27,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:24:27,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/efbea57bb44b46cba0987da16d8a967f is 50, key is test_row_0/A:col10/1731309866486/Put/seqid=0 2024-11-11T07:24:27,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742298_1474 (size=14541) 2024-11-11T07:24:27,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309927168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T07:24:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309927170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,182 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309927171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,182 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T07:24:27,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309927180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T07:24:27,184 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:27,185 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:27,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:27,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309927181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T07:24:27,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309927291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309927291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309927291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309927292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309927302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:27,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:27,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T07:24:27,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309927501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309927512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309927512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309927512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/efbea57bb44b46cba0987da16d8a967f 2024-11-11T07:24:27,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309927521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/ce2cda21c452444794600bbd48fd4504 is 50, key is test_row_0/B:col10/1731309866486/Put/seqid=0 2024-11-11T07:24:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742299_1475 (size=12151) 2024-11-11T07:24:27,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:27,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:27,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T07:24:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309927811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309927832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309927832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309927832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309927841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:27,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:27,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/ce2cda21c452444794600bbd48fd4504 2024-11-11T07:24:27,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/3b81bfe34c864d7790ef45ee4d52a1ef is 50, key is test_row_0/C:col10/1731309866486/Put/seqid=0 2024-11-11T07:24:27,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742300_1476 (size=12151) 2024-11-11T07:24:27,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:27,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:27,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:27,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:28,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T07:24:28,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309928331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/3b81bfe34c864d7790ef45ee4d52a1ef 2024-11-11T07:24:28,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/efbea57bb44b46cba0987da16d8a967f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/efbea57bb44b46cba0987da16d8a967f 2024-11-11T07:24:28,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/efbea57bb44b46cba0987da16d8a967f, entries=200, sequenceid=170, filesize=14.2 K 2024-11-11T07:24:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/ce2cda21c452444794600bbd48fd4504 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/ce2cda21c452444794600bbd48fd4504 2024-11-11T07:24:28,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/ce2cda21c452444794600bbd48fd4504, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T07:24:28,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/3b81bfe34c864d7790ef45ee4d52a1ef as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3b81bfe34c864d7790ef45ee4d52a1ef 2024-11-11T07:24:28,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3b81bfe34c864d7790ef45ee4d52a1ef, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T07:24:28,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 45154c5da223c60a66ac44ba1cc3400c in 1243ms, sequenceid=170, compaction requested=true 2024-11-11T07:24:28,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:28,359 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:28,359 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:28,359 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:28,359 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:28,360 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/A is initiating minor compaction (all files) 2024-11-11T07:24:28,360 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/B is initiating minor compaction (all files) 2024-11-11T07:24:28,360 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/A in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,360 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/B in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,360 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9aff86e17c7d497fb14e3b4b4931aff9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/2235f4c82f2b43f3a0858c98f6dd3d60, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b3d055799ae498db7b41441ebef5ce9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/efbea57bb44b46cba0987da16d8a967f] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=54.4 K 2024-11-11T07:24:28,360 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/919a1342b41241e09d43c7a3044b7465, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c4a86409011341639f69d4f84ad9a06c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/3ac63774f9614eaabb7803d7baff234c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/ce2cda21c452444794600bbd48fd4504] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=47.5 K 2024-11-11T07:24:28,360 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aff86e17c7d497fb14e3b4b4931aff9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731309864376 2024-11-11T07:24:28,360 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 919a1342b41241e09d43c7a3044b7465, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731309864376 2024-11-11T07:24:28,360 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a86409011341639f69d4f84ad9a06c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731309864726 2024-11-11T07:24:28,360 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2235f4c82f2b43f3a0858c98f6dd3d60, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731309864726 2024-11-11T07:24:28,361 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac63774f9614eaabb7803d7baff234c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309864916 2024-11-11T07:24:28,361 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3d055799ae498db7b41441ebef5ce9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309864916 2024-11-11T07:24:28,361 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2cda21c452444794600bbd48fd4504, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731309866486 2024-11-11T07:24:28,361 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbea57bb44b46cba0987da16d8a967f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731309866486 2024-11-11T07:24:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:28,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:24:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:28,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/7112babea61c4525b26a3e5e754f75b4 is 50, key is test_row_0/A:col10/1731309868360/Put/seqid=0 2024-11-11T07:24:28,369 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:28,370 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:28,370 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b04213da3bb47708bc8e88684c88fb7 is 50, key is test_row_0/A:col10/1731309866486/Put/seqid=0 2024-11-11T07:24:28,370 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/32bd06c68c7a449c8bc503c6f0bc1c38 is 50, key is test_row_0/B:col10/1731309866486/Put/seqid=0 2024-11-11T07:24:28,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742301_1477 (size=14541) 2024-11-11T07:24:28,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742302_1478 (size=12527) 2024-11-11T07:24:28,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742303_1479 (size=12527) 2024-11-11T07:24:28,383 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/32bd06c68c7a449c8bc503c6f0bc1c38 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/32bd06c68c7a449c8bc503c6f0bc1c38 2024-11-11T07:24:28,387 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/B of 45154c5da223c60a66ac44ba1cc3400c into 32bd06c68c7a449c8bc503c6f0bc1c38(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:28,387 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:28,387 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/B, priority=12, startTime=1731309868359; duration=0sec 2024-11-11T07:24:28,387 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:28,387 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:B 2024-11-11T07:24:28,387 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:28,388 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:28,388 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/C is initiating minor compaction (all files) 2024-11-11T07:24:28,388 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/C in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,388 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/2a0a0c537c9543149707d5f850abf4a6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3960150445fd4dde94c3316ec91258eb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/bc80bf777d5a4b319033b25fc8d04e1e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3b81bfe34c864d7790ef45ee4d52a1ef] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=47.5 K 2024-11-11T07:24:28,388 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0a0c537c9543149707d5f850abf4a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731309864376 2024-11-11T07:24:28,388 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3960150445fd4dde94c3316ec91258eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731309864726 2024-11-11T07:24:28,388 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting bc80bf777d5a4b319033b25fc8d04e1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731309864916 2024-11-11T07:24:28,389 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b81bfe34c864d7790ef45ee4d52a1ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731309866486 2024-11-11T07:24:28,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309928381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,394 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,395 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/d10311ace9e842dc826cfe449c04869e is 50, key is test_row_0/C:col10/1731309866486/Put/seqid=0 2024-11-11T07:24:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742304_1480 (size=12527) 2024-11-11T07:24:28,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309928385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309928394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,401 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/d10311ace9e842dc826cfe449c04869e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/d10311ace9e842dc826cfe449c04869e 2024-11-11T07:24:28,404 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/C of 45154c5da223c60a66ac44ba1cc3400c into d10311ace9e842dc826cfe449c04869e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:28,404 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:28,404 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/C, priority=12, startTime=1731309868359; duration=0sec 2024-11-11T07:24:28,404 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:28,404 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:C 2024-11-11T07:24:28,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:28,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309928395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309928501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309928501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309928501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309928511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:28,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:28,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:28,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:28,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309928711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309928711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309928711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309928731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:28,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/7112babea61c4525b26a3e5e754f75b4 2024-11-11T07:24:28,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/320a19c9cd8745288f0e889ea82c6827 is 50, key is test_row_0/B:col10/1731309868360/Put/seqid=0 2024-11-11T07:24:28,782 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9b04213da3bb47708bc8e88684c88fb7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b04213da3bb47708bc8e88684c88fb7 2024-11-11T07:24:28,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742305_1481 (size=12151) 2024-11-11T07:24:28,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/320a19c9cd8745288f0e889ea82c6827 2024-11-11T07:24:28,786 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/A of 45154c5da223c60a66ac44ba1cc3400c into 9b04213da3bb47708bc8e88684c88fb7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:28,787 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:28,787 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/A, priority=12, startTime=1731309868358; duration=0sec 2024-11-11T07:24:28,787 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:28,787 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:A 2024-11-11T07:24:28,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/5c0c8a3df40c4471b2d401e9015b1e42 is 50, key is test_row_0/C:col10/1731309868360/Put/seqid=0 2024-11-11T07:24:28,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742306_1482 (size=12151) 2024-11-11T07:24:28,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/5c0c8a3df40c4471b2d401e9015b1e42 2024-11-11T07:24:28,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/7112babea61c4525b26a3e5e754f75b4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7112babea61c4525b26a3e5e754f75b4 2024-11-11T07:24:28,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7112babea61c4525b26a3e5e754f75b4, entries=200, sequenceid=193, filesize=14.2 K 2024-11-11T07:24:28,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/320a19c9cd8745288f0e889ea82c6827 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/320a19c9cd8745288f0e889ea82c6827 2024-11-11T07:24:28,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/320a19c9cd8745288f0e889ea82c6827, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T07:24:28,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/5c0c8a3df40c4471b2d401e9015b1e42 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/5c0c8a3df40c4471b2d401e9015b1e42 2024-11-11T07:24:28,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/5c0c8a3df40c4471b2d401e9015b1e42, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T07:24:28,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 45154c5da223c60a66ac44ba1cc3400c in 445ms, sequenceid=193, compaction requested=false 2024-11-11T07:24:28,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:28,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T07:24:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:28,866 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:24:28,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/077d66481d53423f8475371da9c0f85c is 50, key is test_row_0/A:col10/1731309868392/Put/seqid=0 2024-11-11T07:24:28,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742307_1483 (size=12151) 2024-11-11T07:24:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:29,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309929080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309929082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309929086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309929094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309929202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309929202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309929202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309929202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,275 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/077d66481d53423f8475371da9c0f85c 2024-11-11T07:24:29,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/238a6d0a255b4539bde0ff9f4d293658 is 50, key is test_row_0/B:col10/1731309868392/Put/seqid=0 2024-11-11T07:24:29,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742308_1484 (size=12151) 2024-11-11T07:24:29,285 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/238a6d0a255b4539bde0ff9f4d293658 2024-11-11T07:24:29,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/cb0f9edb7e764741b9557fd6826c2069 is 50, key is test_row_0/C:col10/1731309868392/Put/seqid=0 2024-11-11T07:24:29,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742309_1485 (size=12151) 2024-11-11T07:24:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T07:24:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309929351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,362 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309929412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309929412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309929412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309929412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,696 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/cb0f9edb7e764741b9557fd6826c2069 2024-11-11T07:24:29,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/077d66481d53423f8475371da9c0f85c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/077d66481d53423f8475371da9c0f85c 2024-11-11T07:24:29,703 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/077d66481d53423f8475371da9c0f85c, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:24:29,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/238a6d0a255b4539bde0ff9f4d293658 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/238a6d0a255b4539bde0ff9f4d293658 2024-11-11T07:24:29,706 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/238a6d0a255b4539bde0ff9f4d293658, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:24:29,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/cb0f9edb7e764741b9557fd6826c2069 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/cb0f9edb7e764741b9557fd6826c2069 2024-11-11T07:24:29,710 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/cb0f9edb7e764741b9557fd6826c2069, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:24:29,711 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 45154c5da223c60a66ac44ba1cc3400c in 844ms, sequenceid=209, compaction requested=true 2024-11-11T07:24:29,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:29,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:29,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T07:24:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T07:24:29,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T07:24:29,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5270 sec 2024-11-11T07:24:29,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.5320 sec 2024-11-11T07:24:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:29,726 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:29,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:24:29,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/732ac7edf702484c9b5c2a3b75f757f7 is 50, key is test_row_0/A:col10/1731309869725/Put/seqid=0 2024-11-11T07:24:29,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742310_1486 (size=14541) 2024-11-11T07:24:29,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309929742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309929744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309929751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309929752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309929861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309929862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309929862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:29,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309929871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309930082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309930082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309930082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309930082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/732ac7edf702484c9b5c2a3b75f757f7 2024-11-11T07:24:30,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/1705a62c399f487abbdae2608c26c114 is 50, key is test_row_0/B:col10/1731309869725/Put/seqid=0 2024-11-11T07:24:30,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742311_1487 (size=12151) 2024-11-11T07:24:30,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309930401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309930401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309930401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309930401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/1705a62c399f487abbdae2608c26c114 2024-11-11T07:24:30,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/34dd6bb988174f07ac4d1c99d49f159c is 50, key is test_row_0/C:col10/1731309869725/Put/seqid=0 2024-11-11T07:24:30,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742312_1488 (size=12151) 2024-11-11T07:24:30,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309930911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309930911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309930922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309930922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:30,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/34dd6bb988174f07ac4d1c99d49f159c 2024-11-11T07:24:30,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/732ac7edf702484c9b5c2a3b75f757f7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/732ac7edf702484c9b5c2a3b75f757f7 2024-11-11T07:24:30,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/732ac7edf702484c9b5c2a3b75f757f7, entries=200, sequenceid=233, filesize=14.2 K 2024-11-11T07:24:30,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/1705a62c399f487abbdae2608c26c114 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/1705a62c399f487abbdae2608c26c114 2024-11-11T07:24:30,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/1705a62c399f487abbdae2608c26c114, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T07:24:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/34dd6bb988174f07ac4d1c99d49f159c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/34dd6bb988174f07ac4d1c99d49f159c 2024-11-11T07:24:30,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/34dd6bb988174f07ac4d1c99d49f159c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T07:24:30,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 45154c5da223c60a66ac44ba1cc3400c in 1251ms, sequenceid=233, compaction requested=true 2024-11-11T07:24:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:30,978 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:30,978 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:30,979 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:30,979 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/A is initiating minor compaction (all files) 2024-11-11T07:24:30,979 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/A in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:30,979 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b04213da3bb47708bc8e88684c88fb7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7112babea61c4525b26a3e5e754f75b4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/077d66481d53423f8475371da9c0f85c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/732ac7edf702484c9b5c2a3b75f757f7] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=52.5 K 2024-11-11T07:24:30,980 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:30,980 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b04213da3bb47708bc8e88684c88fb7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731309866486 2024-11-11T07:24:30,980 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/B is initiating minor compaction (all files) 2024-11-11T07:24:30,980 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/B in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:30,981 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/32bd06c68c7a449c8bc503c6f0bc1c38, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/320a19c9cd8745288f0e889ea82c6827, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/238a6d0a255b4539bde0ff9f4d293658, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/1705a62c399f487abbdae2608c26c114] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=47.8 K 2024-11-11T07:24:30,981 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 32bd06c68c7a449c8bc503c6f0bc1c38, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731309866486 2024-11-11T07:24:30,981 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7112babea61c4525b26a3e5e754f75b4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731309867168 2024-11-11T07:24:30,981 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 320a19c9cd8745288f0e889ea82c6827, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731309867168 2024-11-11T07:24:30,981 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 077d66481d53423f8475371da9c0f85c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309868379 2024-11-11T07:24:30,982 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 238a6d0a255b4539bde0ff9f4d293658, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309868379 2024-11-11T07:24:30,982 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732ac7edf702484c9b5c2a3b75f757f7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731309869078 2024-11-11T07:24:30,982 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1705a62c399f487abbdae2608c26c114, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731309869078 2024-11-11T07:24:30,990 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#B#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:30,991 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#A#compaction#416 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:30,991 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/b178315a33774ace8d437aadee96e510 is 50, key is test_row_0/B:col10/1731309869725/Put/seqid=0 2024-11-11T07:24:30,991 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/ad99d96beb2743df8585a07261f9e93b is 50, key is test_row_0/A:col10/1731309869725/Put/seqid=0 2024-11-11T07:24:30,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742313_1489 (size=12663) 2024-11-11T07:24:31,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742314_1490 (size=12663) 2024-11-11T07:24:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T07:24:31,342 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:31,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T07:24:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:24:31,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:31,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:31,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:31,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:31,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:24:31,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/526595bd9e06476084a8c7efaabe4b68 is 50, key is test_row_0/A:col10/1731309869727/Put/seqid=0 2024-11-11T07:24:31,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742315_1491 (size=14541) 2024-11-11T07:24:31,400 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/b178315a33774ace8d437aadee96e510 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b178315a33774ace8d437aadee96e510 2024-11-11T07:24:31,404 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/B of 45154c5da223c60a66ac44ba1cc3400c into b178315a33774ace8d437aadee96e510(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:31,404 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:31,404 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/B, priority=12, startTime=1731309870978; duration=0sec 2024-11-11T07:24:31,404 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:31,404 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:B 2024-11-11T07:24:31,404 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:31,405 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:31,405 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/C is initiating minor compaction (all files) 2024-11-11T07:24:31,405 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/C in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,405 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/d10311ace9e842dc826cfe449c04869e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/5c0c8a3df40c4471b2d401e9015b1e42, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/cb0f9edb7e764741b9557fd6826c2069, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/34dd6bb988174f07ac4d1c99d49f159c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=47.8 K 2024-11-11T07:24:31,406 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting d10311ace9e842dc826cfe449c04869e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731309866486 2024-11-11T07:24:31,406 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c0c8a3df40c4471b2d401e9015b1e42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731309867168 2024-11-11T07:24:31,406 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0f9edb7e764741b9557fd6826c2069, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309868379 2024-11-11T07:24:31,406 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/ad99d96beb2743df8585a07261f9e93b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ad99d96beb2743df8585a07261f9e93b 2024-11-11T07:24:31,406 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 34dd6bb988174f07ac4d1c99d49f159c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731309869078 2024-11-11T07:24:31,411 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/A of 45154c5da223c60a66ac44ba1cc3400c into ad99d96beb2743df8585a07261f9e93b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:31,411 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:31,411 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/A, priority=12, startTime=1731309870978; duration=0sec 2024-11-11T07:24:31,411 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:31,411 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:A 2024-11-11T07:24:31,415 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:31,415 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/c725acd7d0194ddea0436645a283e9cc is 50, key is test_row_0/C:col10/1731309869725/Put/seqid=0 2024-11-11T07:24:31,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742316_1492 (size=12663) 2024-11-11T07:24:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:24:31,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309931539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:31,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309931651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:24:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:31,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/526595bd9e06476084a8c7efaabe4b68 2024-11-11T07:24:31,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/59ddd7d765df411799d3b349889387f6 is 50, key is test_row_0/B:col10/1731309869727/Put/seqid=0 2024-11-11T07:24:31,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742317_1493 (size=12151) 2024-11-11T07:24:31,823 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/c725acd7d0194ddea0436645a283e9cc as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/c725acd7d0194ddea0436645a283e9cc 2024-11-11T07:24:31,826 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/C of 45154c5da223c60a66ac44ba1cc3400c into c725acd7d0194ddea0436645a283e9cc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:31,826 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:31,826 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/C, priority=12, startTime=1731309870978; duration=0sec 2024-11-11T07:24:31,826 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:31,826 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:C 2024-11-11T07:24:31,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:31,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309931871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:31,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309931922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:31,931 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:31,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309931932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:31,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309931932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:31,938 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:31,938 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:31,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309931941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:31,950 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 45154c5da223c60a66ac44ba1cc3400c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:31,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:31,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:31,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:24:32,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309932191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:32,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/59ddd7d765df411799d3b349889387f6 2024-11-11T07:24:32,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/a4482a7d48ba42f483fe056f68fc475d is 50, key is test_row_0/C:col10/1731309869727/Put/seqid=0 2024-11-11T07:24:32,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742318_1494 (size=12151) 2024-11-11T07:24:32,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:32,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:24:32,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/a4482a7d48ba42f483fe056f68fc475d 2024-11-11T07:24:32,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/526595bd9e06476084a8c7efaabe4b68 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/526595bd9e06476084a8c7efaabe4b68 2024-11-11T07:24:32,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/526595bd9e06476084a8c7efaabe4b68, entries=200, sequenceid=246, filesize=14.2 K 2024-11-11T07:24:32,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/59ddd7d765df411799d3b349889387f6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/59ddd7d765df411799d3b349889387f6 2024-11-11T07:24:32,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/59ddd7d765df411799d3b349889387f6, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T07:24:32,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/a4482a7d48ba42f483fe056f68fc475d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/a4482a7d48ba42f483fe056f68fc475d 2024-11-11T07:24:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/a4482a7d48ba42f483fe056f68fc475d, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T07:24:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 45154c5da223c60a66ac44ba1cc3400c in 1244ms, sequenceid=246, compaction requested=false 2024-11-11T07:24:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:32,716 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:32,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:24:32,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:32,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:32,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/37a2986779b1455aab6ae1c2d35609f3 is 50, key is test_row_0/A:col10/1731309871499/Put/seqid=0 2024-11-11T07:24:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742319_1495 (size=14741) 2024-11-11T07:24:32,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/37a2986779b1455aab6ae1c2d35609f3 2024-11-11T07:24:32,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/c1f4ddf07baf48abad1dcccf4b000aef is 50, key is test_row_0/B:col10/1731309871499/Put/seqid=0 2024-11-11T07:24:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742320_1496 (size=12301) 2024-11-11T07:24:32,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309932778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:32,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:32,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309932892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:33,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309933101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:33,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/c1f4ddf07baf48abad1dcccf4b000aef 2024-11-11T07:24:33,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/17ded7db6dc747148ab1fb9c421d5ad5 is 50, key is test_row_0/C:col10/1731309871499/Put/seqid=0 2024-11-11T07:24:33,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742321_1497 (size=12301) 2024-11-11T07:24:33,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:33,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:33,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309933422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:33,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:24:33,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/17ded7db6dc747148ab1fb9c421d5ad5 2024-11-11T07:24:33,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/37a2986779b1455aab6ae1c2d35609f3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/37a2986779b1455aab6ae1c2d35609f3 2024-11-11T07:24:33,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/37a2986779b1455aab6ae1c2d35609f3, entries=200, sequenceid=273, filesize=14.4 K 2024-11-11T07:24:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/c1f4ddf07baf48abad1dcccf4b000aef as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c1f4ddf07baf48abad1dcccf4b000aef 2024-11-11T07:24:33,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c1f4ddf07baf48abad1dcccf4b000aef, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T07:24:33,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/17ded7db6dc747148ab1fb9c421d5ad5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/17ded7db6dc747148ab1fb9c421d5ad5 2024-11-11T07:24:33,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/17ded7db6dc747148ab1fb9c421d5ad5, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T07:24:33,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 45154c5da223c60a66ac44ba1cc3400c in 849ms, sequenceid=273, compaction requested=true 2024-11-11T07:24:33,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:33,565 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:33,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:33,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:33,565 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:33,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:33,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:33,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:33,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:33,566 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:33,566 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/A is initiating minor compaction (all files) 2024-11-11T07:24:33,566 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/A in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,566 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ad99d96beb2743df8585a07261f9e93b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/526595bd9e06476084a8c7efaabe4b68, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/37a2986779b1455aab6ae1c2d35609f3] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=41.0 K 2024-11-11T07:24:33,566 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad99d96beb2743df8585a07261f9e93b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731309869078 2024-11-11T07:24:33,567 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 526595bd9e06476084a8c7efaabe4b68, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731309869727 2024-11-11T07:24:33,567 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:33,567 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/B is initiating minor compaction (all files) 2024-11-11T07:24:33,567 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/B in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,567 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b178315a33774ace8d437aadee96e510, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/59ddd7d765df411799d3b349889387f6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c1f4ddf07baf48abad1dcccf4b000aef] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=36.2 K 2024-11-11T07:24:33,567 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a2986779b1455aab6ae1c2d35609f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309871499 2024-11-11T07:24:33,567 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b178315a33774ace8d437aadee96e510, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731309869078 2024-11-11T07:24:33,568 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 59ddd7d765df411799d3b349889387f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731309869727 2024-11-11T07:24:33,568 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f4ddf07baf48abad1dcccf4b000aef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309871499 2024-11-11T07:24:33,574 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:33,575 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9a296fa75ec54ec8924952953987b8c5 is 50, key is test_row_0/A:col10/1731309871499/Put/seqid=0 2024-11-11T07:24:33,577 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:33,578 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/f036d0ca67a249ec97fa0c2410df1b4d is 50, key is test_row_0/B:col10/1731309871499/Put/seqid=0 2024-11-11T07:24:33,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742323_1499 (size=12915) 2024-11-11T07:24:33,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742322_1498 (size=12915) 2024-11-11T07:24:33,589 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/9a296fa75ec54ec8924952953987b8c5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9a296fa75ec54ec8924952953987b8c5 2024-11-11T07:24:33,593 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/A of 45154c5da223c60a66ac44ba1cc3400c into 9a296fa75ec54ec8924952953987b8c5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:33,593 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:33,593 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/A, priority=13, startTime=1731309873565; duration=0sec 2024-11-11T07:24:33,593 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:33,593 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:A 2024-11-11T07:24:33,593 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:33,594 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:33,594 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/C is initiating minor compaction (all files) 2024-11-11T07:24:33,594 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/C in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,594 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/c725acd7d0194ddea0436645a283e9cc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/a4482a7d48ba42f483fe056f68fc475d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/17ded7db6dc747148ab1fb9c421d5ad5] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=36.2 K 2024-11-11T07:24:33,595 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c725acd7d0194ddea0436645a283e9cc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731309869078 2024-11-11T07:24:33,595 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4482a7d48ba42f483fe056f68fc475d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731309869727 2024-11-11T07:24:33,595 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17ded7db6dc747148ab1fb9c421d5ad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309871499 2024-11-11T07:24:33,603 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:33,604 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/74f5af140fd843708bd8505f662179b8 is 50, key is test_row_0/C:col10/1731309871499/Put/seqid=0 2024-11-11T07:24:33,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742324_1500 (size=12915) 2024-11-11T07:24:33,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:24:33,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:33,637 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:24:33,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/266074b187fd470981cf378e531e4c3c is 50, key is test_row_0/A:col10/1731309872769/Put/seqid=0 2024-11-11T07:24:33,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742325_1501 (size=12301) 2024-11-11T07:24:33,650 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/266074b187fd470981cf378e531e4c3c 2024-11-11T07:24:33,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/a4833f97f03c43dc899d11a46d8c9a8c is 50, key is test_row_0/B:col10/1731309872769/Put/seqid=0 2024-11-11T07:24:33,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742326_1502 (size=12301) 2024-11-11T07:24:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:33,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:33,988 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/f036d0ca67a249ec97fa0c2410df1b4d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/f036d0ca67a249ec97fa0c2410df1b4d 2024-11-11T07:24:34,015 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/74f5af140fd843708bd8505f662179b8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/74f5af140fd843708bd8505f662179b8 2024-11-11T07:24:34,033 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/C of 45154c5da223c60a66ac44ba1cc3400c into 74f5af140fd843708bd8505f662179b8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:34,033 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/B of 45154c5da223c60a66ac44ba1cc3400c into f036d0ca67a249ec97fa0c2410df1b4d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:34,033 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:34,033 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:34,033 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/C, priority=13, startTime=1731309873566; duration=0sec 2024-11-11T07:24:34,033 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/B, priority=13, startTime=1731309873565; duration=0sec 2024-11-11T07:24:34,034 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:34,034 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:C 2024-11-11T07:24:34,034 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:34,034 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:B 2024-11-11T07:24:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309934034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309934034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309934034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309934036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309934036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,062 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/a4833f97f03c43dc899d11a46d8c9a8c 2024-11-11T07:24:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/9e2bbb73e468436996bf668f8148cc8c is 50, key is test_row_0/C:col10/1731309872769/Put/seqid=0 2024-11-11T07:24:34,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742327_1503 (size=12301) 2024-11-11T07:24:34,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309934152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309934152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309934152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309934152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309934152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309934362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309934362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309934362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309934362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309934362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,473 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/9e2bbb73e468436996bf668f8148cc8c 2024-11-11T07:24:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/266074b187fd470981cf378e531e4c3c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/266074b187fd470981cf378e531e4c3c 2024-11-11T07:24:34,479 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/266074b187fd470981cf378e531e4c3c, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T07:24:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/a4833f97f03c43dc899d11a46d8c9a8c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/a4833f97f03c43dc899d11a46d8c9a8c 2024-11-11T07:24:34,483 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/a4833f97f03c43dc899d11a46d8c9a8c, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T07:24:34,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/9e2bbb73e468436996bf668f8148cc8c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/9e2bbb73e468436996bf668f8148cc8c 2024-11-11T07:24:34,487 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/9e2bbb73e468436996bf668f8148cc8c, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T07:24:34,487 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 45154c5da223c60a66ac44ba1cc3400c in 850ms, sequenceid=286, compaction requested=false 2024-11-11T07:24:34,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T07:24:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T07:24:34,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T07:24:34,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1440 sec 2024-11-11T07:24:34,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 3.1480 sec 2024-11-11T07:24:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:34,686 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:34,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:24:34,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/c7cc8dcf4aca40839134b9ff5ab55656 is 50, key is test_row_0/A:col10/1731309873986/Put/seqid=0 2024-11-11T07:24:34,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742328_1504 (size=14741) 2024-11-11T07:24:34,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309934691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309934698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309934699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309934700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309934702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309934811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309934812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309934822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309934822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309934822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309935032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309935032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309935032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309935041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309935041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/c7cc8dcf4aca40839134b9ff5ab55656 2024-11-11T07:24:35,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/8e4ca6d41d984fee99e322cc9fd8758c is 50, key is test_row_0/B:col10/1731309873986/Put/seqid=0 2024-11-11T07:24:35,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742329_1505 (size=12301) 2024-11-11T07:24:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309935342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309935342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309935342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309935351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309935361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/8e4ca6d41d984fee99e322cc9fd8758c 2024-11-11T07:24:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:24:35,512 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:35,512 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:35,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/527da63491044553b5705abc6cfef09e is 50, key is test_row_0/C:col10/1731309873986/Put/seqid=0 2024-11-11T07:24:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T07:24:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:24:35,517 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:35,518 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:35,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:35,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742330_1506 (size=12301) 2024-11-11T07:24:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:24:35,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T07:24:35,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:35,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:35,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:35,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T07:24:35,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:35,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:35,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:24:35,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309935861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309935861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309935862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309935862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309935881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:35,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/527da63491044553b5705abc6cfef09e 2024-11-11T07:24:35,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/c7cc8dcf4aca40839134b9ff5ab55656 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/c7cc8dcf4aca40839134b9ff5ab55656 2024-11-11T07:24:35,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/c7cc8dcf4aca40839134b9ff5ab55656, entries=200, sequenceid=313, filesize=14.4 K 2024-11-11T07:24:35,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/8e4ca6d41d984fee99e322cc9fd8758c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8e4ca6d41d984fee99e322cc9fd8758c 2024-11-11T07:24:35,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8e4ca6d41d984fee99e322cc9fd8758c, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T07:24:35,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/527da63491044553b5705abc6cfef09e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/527da63491044553b5705abc6cfef09e 2024-11-11T07:24:35,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/527da63491044553b5705abc6cfef09e, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T07:24:35,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 45154c5da223c60a66ac44ba1cc3400c in 1248ms, sequenceid=313, compaction requested=true 2024-11-11T07:24:35,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:35,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:35,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:35,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:35,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:35,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:35,935 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:35,935 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:35,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:35,935 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:35,935 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:35,935 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/B is initiating minor compaction (all files) 2024-11-11T07:24:35,935 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/A is initiating minor compaction (all files) 2024-11-11T07:24:35,935 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/B in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,935 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/A in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,936 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9a296fa75ec54ec8924952953987b8c5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/266074b187fd470981cf378e531e4c3c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/c7cc8dcf4aca40839134b9ff5ab55656] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=39.0 K 2024-11-11T07:24:35,936 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/f036d0ca67a249ec97fa0c2410df1b4d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/a4833f97f03c43dc899d11a46d8c9a8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8e4ca6d41d984fee99e322cc9fd8758c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=36.6 K 2024-11-11T07:24:35,936 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a296fa75ec54ec8924952953987b8c5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309871499 2024-11-11T07:24:35,936 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f036d0ca67a249ec97fa0c2410df1b4d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309871499 2024-11-11T07:24:35,936 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a4833f97f03c43dc899d11a46d8c9a8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731309872764 2024-11-11T07:24:35,936 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 266074b187fd470981cf378e531e4c3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731309872764 2024-11-11T07:24:35,936 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4ca6d41d984fee99e322cc9fd8758c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731309873986 2024-11-11T07:24:35,936 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7cc8dcf4aca40839134b9ff5ab55656, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731309873986 2024-11-11T07:24:35,951 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:35,952 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/dc8713f56c0d4eff9b8c4a51f258b911 is 50, key is test_row_0/B:col10/1731309873986/Put/seqid=0 2024-11-11T07:24:35,953 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:35,953 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/7e03a0c27b8c45cb95b4a6d71dc8f146 is 50, key is test_row_0/A:col10/1731309873986/Put/seqid=0 2024-11-11T07:24:35,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742331_1507 (size=13017) 2024-11-11T07:24:35,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742332_1508 (size=13017) 2024-11-11T07:24:35,970 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/dc8713f56c0d4eff9b8c4a51f258b911 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/dc8713f56c0d4eff9b8c4a51f258b911 2024-11-11T07:24:35,977 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/B of 45154c5da223c60a66ac44ba1cc3400c into dc8713f56c0d4eff9b8c4a51f258b911(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:35,977 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:35,977 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/B, priority=13, startTime=1731309875935; duration=0sec 2024-11-11T07:24:35,977 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:35,977 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:B 2024-11-11T07:24:35,977 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:35,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T07:24:35,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,978 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:24:35,979 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:35,979 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/C is initiating minor compaction (all files) 2024-11-11T07:24:35,979 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/C in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:35,979 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/74f5af140fd843708bd8505f662179b8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/9e2bbb73e468436996bf668f8148cc8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/527da63491044553b5705abc6cfef09e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=36.6 K 2024-11-11T07:24:35,979 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f5af140fd843708bd8505f662179b8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731309871499 2024-11-11T07:24:35,979 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2bbb73e468436996bf668f8148cc8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731309872764 2024-11-11T07:24:35,980 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 527da63491044553b5705abc6cfef09e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731309873986 2024-11-11T07:24:35,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/a3756c0187f842eeb1c09a1431c8e51f is 50, key is test_row_0/A:col10/1731309874700/Put/seqid=0 2024-11-11T07:24:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742333_1509 (size=12301) 2024-11-11T07:24:35,998 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:35,999 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/ddc00200f77844ac95ee9310f5ce5378 is 50, key is test_row_0/C:col10/1731309873986/Put/seqid=0 2024-11-11T07:24:36,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742334_1510 (size=13017) 2024-11-11T07:24:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:24:36,373 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/7e03a0c27b8c45cb95b4a6d71dc8f146 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7e03a0c27b8c45cb95b4a6d71dc8f146 2024-11-11T07:24:36,378 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/A of 45154c5da223c60a66ac44ba1cc3400c into 7e03a0c27b8c45cb95b4a6d71dc8f146(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:36,378 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:36,378 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/A, priority=13, startTime=1731309875934; duration=0sec 2024-11-11T07:24:36,378 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:36,378 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:A 2024-11-11T07:24:36,393 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/a3756c0187f842eeb1c09a1431c8e51f 2024-11-11T07:24:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/8d5911952cda40d5ab357fa225cb2533 is 50, key is test_row_0/B:col10/1731309874700/Put/seqid=0 2024-11-11T07:24:36,414 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/ddc00200f77844ac95ee9310f5ce5378 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/ddc00200f77844ac95ee9310f5ce5378 2024-11-11T07:24:36,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742335_1511 (size=12301) 2024-11-11T07:24:36,427 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/8d5911952cda40d5ab357fa225cb2533 2024-11-11T07:24:36,430 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/C of 45154c5da223c60a66ac44ba1cc3400c into ddc00200f77844ac95ee9310f5ce5378(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:36,430 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:36,430 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/C, priority=13, startTime=1731309875935; duration=0sec 2024-11-11T07:24:36,430 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:36,430 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:C 2024-11-11T07:24:36,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/507a6e63c1844a76acf946088f5abd0c is 50, key is test_row_0/C:col10/1731309874700/Put/seqid=0 2024-11-11T07:24:36,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742336_1512 (size=12301) 2024-11-11T07:24:36,501 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/507a6e63c1844a76acf946088f5abd0c 2024-11-11T07:24:36,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/a3756c0187f842eeb1c09a1431c8e51f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a3756c0187f842eeb1c09a1431c8e51f 2024-11-11T07:24:36,517 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a3756c0187f842eeb1c09a1431c8e51f, entries=150, sequenceid=325, filesize=12.0 K 2024-11-11T07:24:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/8d5911952cda40d5ab357fa225cb2533 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8d5911952cda40d5ab357fa225cb2533 2024-11-11T07:24:36,524 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8d5911952cda40d5ab357fa225cb2533, entries=150, sequenceid=325, filesize=12.0 K 2024-11-11T07:24:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/507a6e63c1844a76acf946088f5abd0c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/507a6e63c1844a76acf946088f5abd0c 2024-11-11T07:24:36,528 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:24:36,531 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/507a6e63c1844a76acf946088f5abd0c, entries=150, sequenceid=325, filesize=12.0 K 2024-11-11T07:24:36,532 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 45154c5da223c60a66ac44ba1cc3400c in 554ms, sequenceid=325, compaction requested=false 2024-11-11T07:24:36,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:36,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:36,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T07:24:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T07:24:36,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T07:24:36,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-11-11T07:24:36,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0260 sec 2024-11-11T07:24:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:24:36,651 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:36,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T07:24:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:24:36,656 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:36,657 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:36,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:24:36,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:24:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T07:24:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T07:24:36,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T07:24:36,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-11T07:24:36,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 163 msec 2024-11-11T07:24:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:36,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:24:36,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/92cf33d3e7ce4388b6175d502ee77e7c is 50, key is test_row_0/A:col10/1731309876892/Put/seqid=0 2024-11-11T07:24:36,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742337_1513 (size=17181) 2024-11-11T07:24:36,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309936955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:24:36,971 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:36,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-11T07:24:36,974 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:36,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:36,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:24:36,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309936967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309936967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:36,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:36,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309936969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:36,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309936969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:24:37,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309937081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309937092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309937093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309937093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309937093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:24:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:24:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:24:37,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309937302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309937314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/92cf33d3e7ce4388b6175d502ee77e7c 2024-11-11T07:24:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309937314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309937314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309937314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/261e9532f6464a4e996b0d3897740cb2 is 50, key is test_row_0/B:col10/1731309876892/Put/seqid=0 2024-11-11T07:24:37,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742338_1514 (size=12301) 2024-11-11T07:24:37,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:24:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:24:37,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:24:37,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44142 deadline: 1731309937622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44154 deadline: 1731309937632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44194 deadline: 1731309937632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44204 deadline: 1731309937632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44146 deadline: 1731309937632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 , the old value is region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:24:37,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:24:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. as already flushing 2024-11-11T07:24:37,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:37,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/261e9532f6464a4e996b0d3897740cb2 2024-11-11T07:24:37,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/1c258202da7d4ca68add606a57fc347f is 50, key is test_row_0/C:col10/1731309876892/Put/seqid=0 2024-11-11T07:24:37,810 INFO [Thread-2003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2003. 2024-11-11T07:24:37,810 DEBUG [Thread-2003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:37,810 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,810 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,811 INFO [Thread-2009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2009. 2024-11-11T07:24:37,811 DEBUG [Thread-2009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:37,811 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,812 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:37,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:37,827 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-11-11T07:24:37,827 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:37,827 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,827 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:37,830 INFO [Thread-2011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2011. 2024-11-11T07:24:37,830 DEBUG [Thread-2011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:37,830 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,830 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:37,838 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-11T07:24:37,838 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:37,838 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,838 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:37,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:37,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742339_1515 (size=12301) 2024-11-11T07:24:37,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/1c258202da7d4ca68add606a57fc347f 2024-11-11T07:24:37,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/92cf33d3e7ce4388b6175d502ee77e7c as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/92cf33d3e7ce4388b6175d502ee77e7c 2024-11-11T07:24:37,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/92cf33d3e7ce4388b6175d502ee77e7c, entries=250, sequenceid=339, filesize=16.8 K 2024-11-11T07:24:37,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/261e9532f6464a4e996b0d3897740cb2 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/261e9532f6464a4e996b0d3897740cb2 2024-11-11T07:24:37,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/261e9532f6464a4e996b0d3897740cb2, entries=150, sequenceid=339, filesize=12.0 K 2024-11-11T07:24:37,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/1c258202da7d4ca68add606a57fc347f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1c258202da7d4ca68add606a57fc347f 2024-11-11T07:24:37,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1c258202da7d4ca68add606a57fc347f, entries=150, sequenceid=339, filesize=12.0 K 2024-11-11T07:24:37,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 45154c5da223c60a66ac44ba1cc3400c in 962ms, sequenceid=339, compaction requested=true 2024-11-11T07:24:37,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:37,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:37,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:37,858 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:37,858 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:37,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:37,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:37,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45154c5da223c60a66ac44ba1cc3400c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:37,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:37,859 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:37,859 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:37,859 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/A is initiating minor compaction (all files) 2024-11-11T07:24:37,859 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/B is initiating minor compaction (all files) 2024-11-11T07:24:37,859 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/A in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,859 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/B in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,860 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/dc8713f56c0d4eff9b8c4a51f258b911, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8d5911952cda40d5ab357fa225cb2533, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/261e9532f6464a4e996b0d3897740cb2] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=36.7 K 2024-11-11T07:24:37,860 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7e03a0c27b8c45cb95b4a6d71dc8f146, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a3756c0187f842eeb1c09a1431c8e51f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/92cf33d3e7ce4388b6175d502ee77e7c] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=41.5 K 2024-11-11T07:24:37,860 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e03a0c27b8c45cb95b4a6d71dc8f146, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731309873986 2024-11-11T07:24:37,860 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8713f56c0d4eff9b8c4a51f258b911, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731309873986 2024-11-11T07:24:37,860 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5911952cda40d5ab357fa225cb2533, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731309874688 2024-11-11T07:24:37,860 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3756c0187f842eeb1c09a1431c8e51f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731309874688 2024-11-11T07:24:37,861 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92cf33d3e7ce4388b6175d502ee77e7c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731309876885 2024-11-11T07:24:37,861 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 261e9532f6464a4e996b0d3897740cb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731309876890 2024-11-11T07:24:37,869 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:37,870 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/b81f3fa8ca9741b68c9cde8e47c64202 is 50, key is test_row_0/B:col10/1731309876892/Put/seqid=0 2024-11-11T07:24:37,878 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#A#compaction#444 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:37,879 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/ba91d351942145a1aaf72909abd7b690 is 50, key is test_row_0/A:col10/1731309876892/Put/seqid=0 2024-11-11T07:24:37,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742340_1516 (size=13119) 2024-11-11T07:24:37,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742341_1517 (size=13119) 2024-11-11T07:24:37,892 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/b81f3fa8ca9741b68c9cde8e47c64202 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b81f3fa8ca9741b68c9cde8e47c64202 2024-11-11T07:24:37,897 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/B of 45154c5da223c60a66ac44ba1cc3400c into b81f3fa8ca9741b68c9cde8e47c64202(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:37,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:37,897 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/B, priority=13, startTime=1731309877858; duration=0sec 2024-11-11T07:24:37,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:37,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:B 2024-11-11T07:24:37,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:37,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:24:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,898 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T07:24:37,899 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:37,899 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 45154c5da223c60a66ac44ba1cc3400c/C is initiating minor compaction (all files) 2024-11-11T07:24:37,899 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45154c5da223c60a66ac44ba1cc3400c/C in TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,899 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/ddc00200f77844ac95ee9310f5ce5378, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/507a6e63c1844a76acf946088f5abd0c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1c258202da7d4ca68add606a57fc347f] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp, totalSize=36.7 K 2024-11-11T07:24:37,899 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting ddc00200f77844ac95ee9310f5ce5378, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731309873986 2024-11-11T07:24:37,900 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 507a6e63c1844a76acf946088f5abd0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731309874688 2024-11-11T07:24:37,901 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c258202da7d4ca68add606a57fc347f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731309876890 2024-11-11T07:24:37,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/3b1bd468374b451a81984e5348a7f24b is 50, key is test_row_0/A:col10/1731309876966/Put/seqid=0 2024-11-11T07:24:37,917 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45154c5da223c60a66ac44ba1cc3400c#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:37,918 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/eed340512b3c4fd89e74a7810031cff0 is 50, key is test_row_0/C:col10/1731309876892/Put/seqid=0 2024-11-11T07:24:37,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742342_1518 (size=12301) 2024-11-11T07:24:37,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742343_1519 (size=13119) 2024-11-11T07:24:37,930 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/3b1bd468374b451a81984e5348a7f24b 2024-11-11T07:24:37,938 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/eed340512b3c4fd89e74a7810031cff0 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/eed340512b3c4fd89e74a7810031cff0 2024-11-11T07:24:37,943 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/C of 45154c5da223c60a66ac44ba1cc3400c into eed340512b3c4fd89e74a7810031cff0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:37,943 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:37,943 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/C, priority=13, startTime=1731309877859; duration=0sec 2024-11-11T07:24:37,943 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:37,943 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:C 2024-11-11T07:24:37,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/65e9d92cae544676b3cc09f9217ec287 is 50, key is test_row_0/B:col10/1731309876966/Put/seqid=0 2024-11-11T07:24:37,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742344_1520 (size=12301) 2024-11-11T07:24:37,953 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/65e9d92cae544676b3cc09f9217ec287 2024-11-11T07:24:37,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/1569b5a23bf942ee8e2800e8d5add829 is 50, key is test_row_0/C:col10/1731309876966/Put/seqid=0 2024-11-11T07:24:37,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742345_1521 (size=12301) 2024-11-11T07:24:37,976 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/1569b5a23bf942ee8e2800e8d5add829 2024-11-11T07:24:37,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/3b1bd468374b451a81984e5348a7f24b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3b1bd468374b451a81984e5348a7f24b 2024-11-11T07:24:37,985 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3b1bd468374b451a81984e5348a7f24b, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T07:24:37,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/65e9d92cae544676b3cc09f9217ec287 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/65e9d92cae544676b3cc09f9217ec287 2024-11-11T07:24:37,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/65e9d92cae544676b3cc09f9217ec287, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T07:24:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/1569b5a23bf942ee8e2800e8d5add829 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1569b5a23bf942ee8e2800e8d5add829 2024-11-11T07:24:37,994 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1569b5a23bf942ee8e2800e8d5add829, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T07:24:37,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 45154c5da223c60a66ac44ba1cc3400c in 96ms, sequenceid=366, compaction requested=false 2024-11-11T07:24:37,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:37,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:37,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-11T07:24:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-11T07:24:37,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T07:24:37,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0220 sec 2024-11-11T07:24:38,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0270 sec 2024-11-11T07:24:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:24:38,123 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:38,143 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-11-11T07:24:38,143 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:38,143 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,143 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:38,153 INFO [Thread-1998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1998. 2024-11-11T07:24:38,154 DEBUG [Thread-1998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:38,154 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,154 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-11-11T07:24:38,154 INFO [Thread-1996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1996. 2024-11-11T07:24:38,154 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,154 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:38,154 DEBUG [Thread-1996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:38,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:38,154 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,154 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,154 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,154 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:38,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:38,155 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-11T07:24:38,155 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:24:38,155 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,155 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 870 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2610 rows 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 880 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2637 rows 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 875 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2625 rows 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 887 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2661 rows 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 879 2024-11-11T07:24:38,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2637 rows 2024-11-11T07:24:38,156 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:24:38,156 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:24:38,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:38,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:24:38,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:24:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:24:38,159 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309878159"}]},"ts":"1731309878159"} 2024-11-11T07:24:38,160 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:24:38,160 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:24:38,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:24:38,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, UNASSIGN}] 2024-11-11T07:24:38,162 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, UNASSIGN 2024-11-11T07:24:38,163 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=45154c5da223c60a66ac44ba1cc3400c, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:38,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, UNASSIGN because future has completed 2024-11-11T07:24:38,164 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:24:38,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:24:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:24:38,304 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/ba91d351942145a1aaf72909abd7b690 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ba91d351942145a1aaf72909abd7b690 2024-11-11T07:24:38,309 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45154c5da223c60a66ac44ba1cc3400c/A of 45154c5da223c60a66ac44ba1cc3400c into ba91d351942145a1aaf72909abd7b690(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:38,309 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45154c5da223c60a66ac44ba1cc3400c: 2024-11-11T07:24:38,309 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., storeName=45154c5da223c60a66ac44ba1cc3400c/A, priority=13, startTime=1731309877858; duration=0sec 2024-11-11T07:24:38,309 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:38,309 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45154c5da223c60a66ac44ba1cc3400c:A 2024-11-11T07:24:38,316 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:38,316 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:24:38,316 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 45154c5da223c60a66ac44ba1cc3400c, disabling compactions & flushes 2024-11-11T07:24:38,316 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:38,316 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:38,316 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. after waiting 0 ms 2024-11-11T07:24:38,316 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:38,316 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 45154c5da223c60a66ac44ba1cc3400c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T07:24:38,320 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/5bb4af4dfa124cafbac5a228c3a1cece is 50, key is test_row_0/A:col10/1731309878153/Put/seqid=0 2024-11-11T07:24:38,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742346_1522 (size=12301) 2024-11-11T07:24:38,326 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/5bb4af4dfa124cafbac5a228c3a1cece 2024-11-11T07:24:38,332 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/44dd7e36797948478b3d12be83299e66 is 50, key is test_row_0/B:col10/1731309878153/Put/seqid=0 2024-11-11T07:24:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742347_1523 (size=12301) 2024-11-11T07:24:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:24:38,736 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/44dd7e36797948478b3d12be83299e66 2024-11-11T07:24:38,742 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/025c6277fe2548bab04f96e22101f7ec is 50, key is test_row_0/C:col10/1731309878153/Put/seqid=0 2024-11-11T07:24:38,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742348_1524 (size=12301) 2024-11-11T07:24:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:24:39,147 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/025c6277fe2548bab04f96e22101f7ec 2024-11-11T07:24:39,152 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/A/5bb4af4dfa124cafbac5a228c3a1cece as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/5bb4af4dfa124cafbac5a228c3a1cece 2024-11-11T07:24:39,156 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/5bb4af4dfa124cafbac5a228c3a1cece, entries=150, sequenceid=376, filesize=12.0 K 2024-11-11T07:24:39,157 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/B/44dd7e36797948478b3d12be83299e66 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/44dd7e36797948478b3d12be83299e66 2024-11-11T07:24:39,160 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/44dd7e36797948478b3d12be83299e66, entries=150, sequenceid=376, filesize=12.0 K 2024-11-11T07:24:39,161 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/.tmp/C/025c6277fe2548bab04f96e22101f7ec as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/025c6277fe2548bab04f96e22101f7ec 2024-11-11T07:24:39,165 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/025c6277fe2548bab04f96e22101f7ec, entries=150, sequenceid=376, filesize=12.0 K 2024-11-11T07:24:39,166 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 45154c5da223c60a66ac44ba1cc3400c in 850ms, sequenceid=376, compaction requested=true 2024-11-11T07:24:39,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9208737c389d4ede9cc4927f989e47cf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b5c3bdd4d1941849386a90aea39b66d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a89d0d1717fa4c5db38b60e43fd40308, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/093a475914e947da9adc60ab1d21077f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3f92bbfe1d75480e91cd1b7c4538687c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bec8ad8e106345869ba36513f3326e84, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bb17a0729a264ceabf850b0655ec9a2f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9aff86e17c7d497fb14e3b4b4931aff9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/2235f4c82f2b43f3a0858c98f6dd3d60, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b3d055799ae498db7b41441ebef5ce9, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/efbea57bb44b46cba0987da16d8a967f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b04213da3bb47708bc8e88684c88fb7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7112babea61c4525b26a3e5e754f75b4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/077d66481d53423f8475371da9c0f85c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/732ac7edf702484c9b5c2a3b75f757f7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ad99d96beb2743df8585a07261f9e93b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/526595bd9e06476084a8c7efaabe4b68, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/37a2986779b1455aab6ae1c2d35609f3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9a296fa75ec54ec8924952953987b8c5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/266074b187fd470981cf378e531e4c3c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/c7cc8dcf4aca40839134b9ff5ab55656, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7e03a0c27b8c45cb95b4a6d71dc8f146, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a3756c0187f842eeb1c09a1431c8e51f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/92cf33d3e7ce4388b6175d502ee77e7c] to archive 2024-11-11T07:24:39,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:24:39,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9208737c389d4ede9cc4927f989e47cf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9208737c389d4ede9cc4927f989e47cf 2024-11-11T07:24:39,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b5c3bdd4d1941849386a90aea39b66d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b5c3bdd4d1941849386a90aea39b66d 2024-11-11T07:24:39,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a89d0d1717fa4c5db38b60e43fd40308 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a89d0d1717fa4c5db38b60e43fd40308 2024-11-11T07:24:39,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/093a475914e947da9adc60ab1d21077f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/093a475914e947da9adc60ab1d21077f 2024-11-11T07:24:39,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3f92bbfe1d75480e91cd1b7c4538687c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3f92bbfe1d75480e91cd1b7c4538687c 2024-11-11T07:24:39,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bec8ad8e106345869ba36513f3326e84 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bec8ad8e106345869ba36513f3326e84 2024-11-11T07:24:39,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bb17a0729a264ceabf850b0655ec9a2f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/bb17a0729a264ceabf850b0655ec9a2f 2024-11-11T07:24:39,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9aff86e17c7d497fb14e3b4b4931aff9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9aff86e17c7d497fb14e3b4b4931aff9 2024-11-11T07:24:39,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/2235f4c82f2b43f3a0858c98f6dd3d60 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/2235f4c82f2b43f3a0858c98f6dd3d60 2024-11-11T07:24:39,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b3d055799ae498db7b41441ebef5ce9 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b3d055799ae498db7b41441ebef5ce9 2024-11-11T07:24:39,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/efbea57bb44b46cba0987da16d8a967f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/efbea57bb44b46cba0987da16d8a967f 2024-11-11T07:24:39,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b04213da3bb47708bc8e88684c88fb7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9b04213da3bb47708bc8e88684c88fb7 2024-11-11T07:24:39,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7112babea61c4525b26a3e5e754f75b4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7112babea61c4525b26a3e5e754f75b4 2024-11-11T07:24:39,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/077d66481d53423f8475371da9c0f85c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/077d66481d53423f8475371da9c0f85c 2024-11-11T07:24:39,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/732ac7edf702484c9b5c2a3b75f757f7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/732ac7edf702484c9b5c2a3b75f757f7 2024-11-11T07:24:39,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ad99d96beb2743df8585a07261f9e93b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ad99d96beb2743df8585a07261f9e93b 2024-11-11T07:24:39,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/526595bd9e06476084a8c7efaabe4b68 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/526595bd9e06476084a8c7efaabe4b68 2024-11-11T07:24:39,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/37a2986779b1455aab6ae1c2d35609f3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/37a2986779b1455aab6ae1c2d35609f3 2024-11-11T07:24:39,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9a296fa75ec54ec8924952953987b8c5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/9a296fa75ec54ec8924952953987b8c5 2024-11-11T07:24:39,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/266074b187fd470981cf378e531e4c3c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/266074b187fd470981cf378e531e4c3c 2024-11-11T07:24:39,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/c7cc8dcf4aca40839134b9ff5ab55656 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/c7cc8dcf4aca40839134b9ff5ab55656 2024-11-11T07:24:39,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7e03a0c27b8c45cb95b4a6d71dc8f146 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/7e03a0c27b8c45cb95b4a6d71dc8f146 2024-11-11T07:24:39,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a3756c0187f842eeb1c09a1431c8e51f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/a3756c0187f842eeb1c09a1431c8e51f 2024-11-11T07:24:39,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/92cf33d3e7ce4388b6175d502ee77e7c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/92cf33d3e7ce4388b6175d502ee77e7c 2024-11-11T07:24:39,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/583a9a9ed90b46a09288b1ae12ceb258, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/0cbd102f38d9431a9900eaf0c988ea16, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/545d05495a7e4578b0b5b68f95b92e85, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/eb1c0419a61141caa8eae52ba0ac0265, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/089a82fc1baf4726995b46faf6e64ba4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/6e13de21e17744699a6506653d7d7e71, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/919a1342b41241e09d43c7a3044b7465, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/cbbb7d8de70149e1a050e2872e0a9ac0, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c4a86409011341639f69d4f84ad9a06c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/3ac63774f9614eaabb7803d7baff234c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/32bd06c68c7a449c8bc503c6f0bc1c38, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/ce2cda21c452444794600bbd48fd4504, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/320a19c9cd8745288f0e889ea82c6827, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/238a6d0a255b4539bde0ff9f4d293658, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b178315a33774ace8d437aadee96e510, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/1705a62c399f487abbdae2608c26c114, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/59ddd7d765df411799d3b349889387f6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/f036d0ca67a249ec97fa0c2410df1b4d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c1f4ddf07baf48abad1dcccf4b000aef, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/a4833f97f03c43dc899d11a46d8c9a8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/dc8713f56c0d4eff9b8c4a51f258b911, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8e4ca6d41d984fee99e322cc9fd8758c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8d5911952cda40d5ab357fa225cb2533, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/261e9532f6464a4e996b0d3897740cb2] to archive 2024-11-11T07:24:39,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:24:39,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/583a9a9ed90b46a09288b1ae12ceb258 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/583a9a9ed90b46a09288b1ae12ceb258 2024-11-11T07:24:39,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/0cbd102f38d9431a9900eaf0c988ea16 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/0cbd102f38d9431a9900eaf0c988ea16 2024-11-11T07:24:39,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/545d05495a7e4578b0b5b68f95b92e85 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/545d05495a7e4578b0b5b68f95b92e85 2024-11-11T07:24:39,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/eb1c0419a61141caa8eae52ba0ac0265 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/eb1c0419a61141caa8eae52ba0ac0265 2024-11-11T07:24:39,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/089a82fc1baf4726995b46faf6e64ba4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/089a82fc1baf4726995b46faf6e64ba4 2024-11-11T07:24:39,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/6e13de21e17744699a6506653d7d7e71 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/6e13de21e17744699a6506653d7d7e71 2024-11-11T07:24:39,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/919a1342b41241e09d43c7a3044b7465 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/919a1342b41241e09d43c7a3044b7465 2024-11-11T07:24:39,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/cbbb7d8de70149e1a050e2872e0a9ac0 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/cbbb7d8de70149e1a050e2872e0a9ac0 2024-11-11T07:24:39,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c4a86409011341639f69d4f84ad9a06c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c4a86409011341639f69d4f84ad9a06c 2024-11-11T07:24:39,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/3ac63774f9614eaabb7803d7baff234c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/3ac63774f9614eaabb7803d7baff234c 2024-11-11T07:24:39,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/32bd06c68c7a449c8bc503c6f0bc1c38 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/32bd06c68c7a449c8bc503c6f0bc1c38 2024-11-11T07:24:39,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/ce2cda21c452444794600bbd48fd4504 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/ce2cda21c452444794600bbd48fd4504 2024-11-11T07:24:39,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/320a19c9cd8745288f0e889ea82c6827 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/320a19c9cd8745288f0e889ea82c6827 2024-11-11T07:24:39,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/238a6d0a255b4539bde0ff9f4d293658 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/238a6d0a255b4539bde0ff9f4d293658 2024-11-11T07:24:39,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b178315a33774ace8d437aadee96e510 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b178315a33774ace8d437aadee96e510 2024-11-11T07:24:39,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/1705a62c399f487abbdae2608c26c114 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/1705a62c399f487abbdae2608c26c114 2024-11-11T07:24:39,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/59ddd7d765df411799d3b349889387f6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/59ddd7d765df411799d3b349889387f6 2024-11-11T07:24:39,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/f036d0ca67a249ec97fa0c2410df1b4d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/f036d0ca67a249ec97fa0c2410df1b4d 2024-11-11T07:24:39,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c1f4ddf07baf48abad1dcccf4b000aef to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/c1f4ddf07baf48abad1dcccf4b000aef 2024-11-11T07:24:39,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/a4833f97f03c43dc899d11a46d8c9a8c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/a4833f97f03c43dc899d11a46d8c9a8c 2024-11-11T07:24:39,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/dc8713f56c0d4eff9b8c4a51f258b911 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/dc8713f56c0d4eff9b8c4a51f258b911 2024-11-11T07:24:39,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8e4ca6d41d984fee99e322cc9fd8758c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8e4ca6d41d984fee99e322cc9fd8758c 2024-11-11T07:24:39,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8d5911952cda40d5ab357fa225cb2533 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/8d5911952cda40d5ab357fa225cb2533 2024-11-11T07:24:39,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/261e9532f6464a4e996b0d3897740cb2 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/261e9532f6464a4e996b0d3897740cb2 2024-11-11T07:24:39,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/0bc115a497494b72ae0ea633982c2fdd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/99750b9528014c12a0c17b32a6bffaee, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/048518a36e11418b858aa615630a230d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/03775f2f6ee04c2b8f1787346fb5ecf5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b0089256976749d0b4b0eaae3511558a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b3e2eeeb68a14f97bd5ae29d45f0af34, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/2a0a0c537c9543149707d5f850abf4a6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/afc69ad7db1c43f58fd1a9b464596c99, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3960150445fd4dde94c3316ec91258eb, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/bc80bf777d5a4b319033b25fc8d04e1e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/d10311ace9e842dc826cfe449c04869e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3b81bfe34c864d7790ef45ee4d52a1ef, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/5c0c8a3df40c4471b2d401e9015b1e42, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/cb0f9edb7e764741b9557fd6826c2069, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/c725acd7d0194ddea0436645a283e9cc, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/34dd6bb988174f07ac4d1c99d49f159c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/a4482a7d48ba42f483fe056f68fc475d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/74f5af140fd843708bd8505f662179b8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/17ded7db6dc747148ab1fb9c421d5ad5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/9e2bbb73e468436996bf668f8148cc8c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/ddc00200f77844ac95ee9310f5ce5378, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/527da63491044553b5705abc6cfef09e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/507a6e63c1844a76acf946088f5abd0c, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1c258202da7d4ca68add606a57fc347f] to archive 2024-11-11T07:24:39,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:24:39,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/0bc115a497494b72ae0ea633982c2fdd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/0bc115a497494b72ae0ea633982c2fdd 2024-11-11T07:24:39,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/99750b9528014c12a0c17b32a6bffaee to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/99750b9528014c12a0c17b32a6bffaee 2024-11-11T07:24:39,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/048518a36e11418b858aa615630a230d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/048518a36e11418b858aa615630a230d 2024-11-11T07:24:39,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/03775f2f6ee04c2b8f1787346fb5ecf5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/03775f2f6ee04c2b8f1787346fb5ecf5 2024-11-11T07:24:39,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b0089256976749d0b4b0eaae3511558a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b0089256976749d0b4b0eaae3511558a 2024-11-11T07:24:39,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b3e2eeeb68a14f97bd5ae29d45f0af34 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/b3e2eeeb68a14f97bd5ae29d45f0af34 2024-11-11T07:24:39,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/2a0a0c537c9543149707d5f850abf4a6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/2a0a0c537c9543149707d5f850abf4a6 2024-11-11T07:24:39,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/afc69ad7db1c43f58fd1a9b464596c99 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/afc69ad7db1c43f58fd1a9b464596c99 2024-11-11T07:24:39,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3960150445fd4dde94c3316ec91258eb to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3960150445fd4dde94c3316ec91258eb 2024-11-11T07:24:39,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/bc80bf777d5a4b319033b25fc8d04e1e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/bc80bf777d5a4b319033b25fc8d04e1e 2024-11-11T07:24:39,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/d10311ace9e842dc826cfe449c04869e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/d10311ace9e842dc826cfe449c04869e 2024-11-11T07:24:39,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3b81bfe34c864d7790ef45ee4d52a1ef to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/3b81bfe34c864d7790ef45ee4d52a1ef 2024-11-11T07:24:39,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/5c0c8a3df40c4471b2d401e9015b1e42 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/5c0c8a3df40c4471b2d401e9015b1e42 2024-11-11T07:24:39,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/cb0f9edb7e764741b9557fd6826c2069 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/cb0f9edb7e764741b9557fd6826c2069 2024-11-11T07:24:39,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/c725acd7d0194ddea0436645a283e9cc to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/c725acd7d0194ddea0436645a283e9cc 2024-11-11T07:24:39,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/34dd6bb988174f07ac4d1c99d49f159c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/34dd6bb988174f07ac4d1c99d49f159c 2024-11-11T07:24:39,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/a4482a7d48ba42f483fe056f68fc475d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/a4482a7d48ba42f483fe056f68fc475d 2024-11-11T07:24:39,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/74f5af140fd843708bd8505f662179b8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/74f5af140fd843708bd8505f662179b8 2024-11-11T07:24:39,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/17ded7db6dc747148ab1fb9c421d5ad5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/17ded7db6dc747148ab1fb9c421d5ad5 2024-11-11T07:24:39,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/9e2bbb73e468436996bf668f8148cc8c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/9e2bbb73e468436996bf668f8148cc8c 2024-11-11T07:24:39,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/ddc00200f77844ac95ee9310f5ce5378 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/ddc00200f77844ac95ee9310f5ce5378 2024-11-11T07:24:39,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/527da63491044553b5705abc6cfef09e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/527da63491044553b5705abc6cfef09e 2024-11-11T07:24:39,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/507a6e63c1844a76acf946088f5abd0c to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/507a6e63c1844a76acf946088f5abd0c 2024-11-11T07:24:39,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1c258202da7d4ca68add606a57fc347f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1c258202da7d4ca68add606a57fc347f 2024-11-11T07:24:39,240 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/recovered.edits/379.seqid, newMaxSeqId=379, maxSeqId=1 2024-11-11T07:24:39,241 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. 2024-11-11T07:24:39,241 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 45154c5da223c60a66ac44ba1cc3400c: Waiting for close lock at 1731309878316Running coprocessor pre-close hooks at 1731309878316Disabling compacts and flushes for region at 1731309878316Disabling writes for close at 1731309878316Obtaining lock to block concurrent updates at 1731309878316Preparing flush snapshotting stores in 45154c5da223c60a66ac44ba1cc3400c at 1731309878316Finished memstore snapshotting TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731309878317 (+1 ms)Flushing stores of TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c. at 1731309878317Flushing 45154c5da223c60a66ac44ba1cc3400c/A: creating writer at 1731309878317Flushing 45154c5da223c60a66ac44ba1cc3400c/A: appending metadata at 1731309878320 (+3 ms)Flushing 45154c5da223c60a66ac44ba1cc3400c/A: closing flushed file at 1731309878320Flushing 45154c5da223c60a66ac44ba1cc3400c/B: creating writer at 1731309878329 (+9 ms)Flushing 45154c5da223c60a66ac44ba1cc3400c/B: appending metadata at 1731309878332 (+3 ms)Flushing 45154c5da223c60a66ac44ba1cc3400c/B: closing flushed file at 1731309878332Flushing 45154c5da223c60a66ac44ba1cc3400c/C: creating writer at 1731309878739 (+407 ms)Flushing 45154c5da223c60a66ac44ba1cc3400c/C: appending metadata at 1731309878742 (+3 ms)Flushing 45154c5da223c60a66ac44ba1cc3400c/C: closing flushed file at 1731309878742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5567f287: reopening flushed file at 1731309879151 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e03102c: reopening flushed file at 1731309879156 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c741eb: reopening flushed file at 1731309879160 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 45154c5da223c60a66ac44ba1cc3400c in 850ms, sequenceid=376, compaction requested=true at 1731309879166 (+6 ms)Writing region close event to WAL at 1731309879238 (+72 ms)Running coprocessor post-close hooks at 1731309879241 (+3 ms)Closed at 1731309879241 2024-11-11T07:24:39,243 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:39,243 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=45154c5da223c60a66ac44ba1cc3400c, regionState=CLOSED 2024-11-11T07:24:39,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:24:39,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T07:24:39,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 45154c5da223c60a66ac44ba1cc3400c, server=f334e8b0fbdf,39329,1731309728191 in 1.0810 sec 2024-11-11T07:24:39,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-11T07:24:39,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45154c5da223c60a66ac44ba1cc3400c, UNASSIGN in 1.0850 sec 2024-11-11T07:24:39,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T07:24:39,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0880 sec 2024-11-11T07:24:39,253 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309879252"}]},"ts":"1731309879252"} 2024-11-11T07:24:39,254 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:24:39,254 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:24:39,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0980 sec 2024-11-11T07:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:24:39,292 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:39,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:24:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:39,294 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T07:24:39,295 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:39,296 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:39,298 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/recovered.edits] 2024-11-11T07:24:39,300 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3b1bd468374b451a81984e5348a7f24b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/3b1bd468374b451a81984e5348a7f24b 2024-11-11T07:24:39,301 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/5bb4af4dfa124cafbac5a228c3a1cece to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/5bb4af4dfa124cafbac5a228c3a1cece 2024-11-11T07:24:39,302 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ba91d351942145a1aaf72909abd7b690 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/A/ba91d351942145a1aaf72909abd7b690 2024-11-11T07:24:39,304 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/44dd7e36797948478b3d12be83299e66 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/44dd7e36797948478b3d12be83299e66 2024-11-11T07:24:39,305 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/65e9d92cae544676b3cc09f9217ec287 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/65e9d92cae544676b3cc09f9217ec287 2024-11-11T07:24:39,306 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b81f3fa8ca9741b68c9cde8e47c64202 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/B/b81f3fa8ca9741b68c9cde8e47c64202 2024-11-11T07:24:39,308 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/025c6277fe2548bab04f96e22101f7ec to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/025c6277fe2548bab04f96e22101f7ec 2024-11-11T07:24:39,309 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1569b5a23bf942ee8e2800e8d5add829 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/1569b5a23bf942ee8e2800e8d5add829 2024-11-11T07:24:39,310 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/eed340512b3c4fd89e74a7810031cff0 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/C/eed340512b3c4fd89e74a7810031cff0 2024-11-11T07:24:39,312 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/recovered.edits/379.seqid to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c/recovered.edits/379.seqid 2024-11-11T07:24:39,312 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/45154c5da223c60a66ac44ba1cc3400c 2024-11-11T07:24:39,312 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:24:39,314 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:39,316 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:24:39,318 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:24:39,319 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:39,319 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:24:39,319 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731309879319"}]},"ts":"9223372036854775807"} 2024-11-11T07:24:39,320 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:24:39,320 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 45154c5da223c60a66ac44ba1cc3400c, NAME => 'TestAcidGuarantees,,1731309855947.45154c5da223c60a66ac44ba1cc3400c.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:24:39,321 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:24:39,321 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731309879321"}]},"ts":"9223372036854775807"} 2024-11-11T07:24:39,323 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:24:39,323 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:39,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-11T07:24:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T07:24:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:24:39,402 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:39,412 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1950 (was 1453) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8962 (was 7900) - AvailableMemoryMB LEAK? - 2024-11-11T07:24:39,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1950, ProcessCount=11, AvailableMemoryMB=8956 2024-11-11T07:24:39,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:24:39,423 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:24:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:39,425 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:24:39,425 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:39,425 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-11T07:24:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T07:24:39,426 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:24:39,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742349_1525 (size=959) 2024-11-11T07:24:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T07:24:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T07:24:39,840 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537 2024-11-11T07:24:39,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742350_1526 (size=53) 2024-11-11T07:24:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T07:24:40,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:24:40,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 022a345c4df8dd612d037e1eac2d4556, disabling compactions & flushes 2024-11-11T07:24:40,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:40,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:40,246 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. after waiting 0 ms 2024-11-11T07:24:40,246 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:40,246 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:40,246 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 022a345c4df8dd612d037e1eac2d4556: Waiting for close lock at 1731309880245Disabling compacts and flushes for region at 1731309880245Disabling writes for close at 1731309880246 (+1 ms)Writing region close event to WAL at 1731309880246Closed at 1731309880246 2024-11-11T07:24:40,247 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:24:40,247 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731309880247"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731309880247"}]},"ts":"1731309880247"} 2024-11-11T07:24:40,249 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:24:40,250 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:24:40,250 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309880250"}]},"ts":"1731309880250"} 2024-11-11T07:24:40,251 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:24:40,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, ASSIGN}] 2024-11-11T07:24:40,253 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, ASSIGN 2024-11-11T07:24:40,254 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, ASSIGN; state=OFFLINE, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=false 2024-11-11T07:24:40,404 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:40,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, ASSIGN because future has completed 2024-11-11T07:24:40,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:24:40,561 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:40,561 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:24:40,561 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,561 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:24:40,561 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,561 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T07:24:40,563 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,564 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 022a345c4df8dd612d037e1eac2d4556 columnFamilyName A 2024-11-11T07:24:40,564 DEBUG [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:40,564 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(327): Store=022a345c4df8dd612d037e1eac2d4556/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:40,564 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,566 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 022a345c4df8dd612d037e1eac2d4556 columnFamilyName B 2024-11-11T07:24:40,566 DEBUG [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:40,566 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(327): Store=022a345c4df8dd612d037e1eac2d4556/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:40,566 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,567 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 022a345c4df8dd612d037e1eac2d4556 columnFamilyName C 2024-11-11T07:24:40,567 DEBUG [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:40,567 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(327): Store=022a345c4df8dd612d037e1eac2d4556/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:40,568 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,568 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,568 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,569 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,569 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,570 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:24:40,571 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,573 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:24:40,573 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 022a345c4df8dd612d037e1eac2d4556; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64516727, jitterRate=-0.03862585127353668}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:24:40,573 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:40,574 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 022a345c4df8dd612d037e1eac2d4556: Running coprocessor pre-open hook at 1731309880562Writing region info on filesystem at 1731309880562Initializing all the Stores at 1731309880562Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309880562Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309880562Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309880562Cleaning up temporary data from old regions at 1731309880569 (+7 ms)Running coprocessor post-open hooks at 1731309880573 (+4 ms)Region opened successfully at 1731309880574 (+1 ms) 2024-11-11T07:24:40,574 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., pid=140, masterSystemTime=1731309880558 2024-11-11T07:24:40,576 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:40,576 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:40,576 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=OPEN, openSeqNum=2, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:40,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:24:40,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T07:24:40,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 in 172 msec 2024-11-11T07:24:40,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-11T07:24:40,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, ASSIGN in 328 msec 2024-11-11T07:24:40,582 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:24:40,582 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309880582"}]},"ts":"1731309880582"} 2024-11-11T07:24:40,583 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:24:40,584 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:24:40,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-11T07:24:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T07:24:41,572 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:41,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658b2e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:41,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:41,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:41,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38955848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:41,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513d7d35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:41,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:41,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:41,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:24:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f334e8b0fbdf,39865,1731309727465 2024-11-11T07:24:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f89b3aa 2024-11-11T07:24:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:24:41,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:24:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:24:41,580 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:24:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T07:24:41,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742351_1527 (size=995) 2024-11-11T07:24:41,991 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T07:24:41,991 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T07:24:41,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:24:41,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, REOPEN/MOVE}] 2024-11-11T07:24:41,996 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, REOPEN/MOVE 2024-11-11T07:24:41,997 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:41,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, REOPEN/MOVE because future has completed 2024-11-11T07:24:41,998 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:24:41,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:24:42,150 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,150 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:24:42,151 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 022a345c4df8dd612d037e1eac2d4556, disabling compactions & flushes 2024-11-11T07:24:42,151 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,151 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,151 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. after waiting 0 ms 2024-11-11T07:24:42,151 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,154 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T07:24:42,154 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,154 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 022a345c4df8dd612d037e1eac2d4556: Waiting for close lock at 1731309882150Running coprocessor pre-close hooks at 1731309882150Disabling compacts and flushes for region at 1731309882150Disabling writes for close at 1731309882151 (+1 ms)Writing region close event to WAL at 1731309882151Running coprocessor post-close hooks at 1731309882154 (+3 ms)Closed at 1731309882154 2024-11-11T07:24:42,154 WARN [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 022a345c4df8dd612d037e1eac2d4556 to self. 2024-11-11T07:24:42,156 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,156 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=CLOSED 2024-11-11T07:24:42,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:24:42,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-11T07:24:42,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 in 160 msec 2024-11-11T07:24:42,160 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, REOPEN/MOVE; state=CLOSED, location=f334e8b0fbdf,39329,1731309728191; forceNewPlan=false, retain=true 2024-11-11T07:24:42,310 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=OPENING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, REOPEN/MOVE because future has completed 2024-11-11T07:24:42,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:24:42,466 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,466 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:24:42,466 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,467 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:24:42,467 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,467 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,468 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,469 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 022a345c4df8dd612d037e1eac2d4556 columnFamilyName A 2024-11-11T07:24:42,470 DEBUG [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:42,470 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(327): Store=022a345c4df8dd612d037e1eac2d4556/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:42,470 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,471 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 022a345c4df8dd612d037e1eac2d4556 columnFamilyName B 2024-11-11T07:24:42,471 DEBUG [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:42,472 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(327): Store=022a345c4df8dd612d037e1eac2d4556/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:42,472 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,472 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 022a345c4df8dd612d037e1eac2d4556 columnFamilyName C 2024-11-11T07:24:42,472 DEBUG [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:42,473 INFO [StoreOpener-022a345c4df8dd612d037e1eac2d4556-1 {}] regionserver.HStore(327): Store=022a345c4df8dd612d037e1eac2d4556/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:24:42,473 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,473 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,474 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,475 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,475 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,475 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:24:42,477 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,477 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 022a345c4df8dd612d037e1eac2d4556; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73156540, jitterRate=0.09011739492416382}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:24:42,477 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,478 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 022a345c4df8dd612d037e1eac2d4556: Running coprocessor pre-open hook at 1731309882467Writing region info on filesystem at 1731309882467Initializing all the Stores at 1731309882467Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309882467Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309882468 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731309882468Cleaning up temporary data from old regions at 1731309882475 (+7 ms)Running coprocessor post-open hooks at 1731309882478 (+3 ms)Region opened successfully at 1731309882478 2024-11-11T07:24:42,479 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., pid=145, masterSystemTime=1731309882463 2024-11-11T07:24:42,480 DEBUG [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,480 INFO [RS_OPEN_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,481 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=OPEN, openSeqNum=5, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:24:42,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-11T07:24:42,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 in 171 msec 2024-11-11T07:24:42,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-11T07:24:42,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, REOPEN/MOVE in 489 msec 2024-11-11T07:24:42,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T07:24:42,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-11T07:24:42,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-11T07:24:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T07:24:42,491 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:42,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5581e3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,492 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233b6a88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,494 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db85b39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,495 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6868186b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,496 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@668265a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,499 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b11dfed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7861000f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,502 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad5836b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,503 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15dc2c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,504 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cfcc0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adec4c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@863a55c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@534e6722, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,509 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f774b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,510 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6d5c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,512 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28b2b20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,513 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b66fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,515 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d169e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,516 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b2c947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f334e8b0fbdf,39865,-1 for getting cluster id 2024-11-11T07:24:42,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:24:42,518 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81636d0b-cdc2-4858-a30b-4ad347eabbaf' 2024-11-11T07:24:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:24:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81636d0b-cdc2-4858-a30b-4ad347eabbaf" 2024-11-11T07:24:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5107175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f334e8b0fbdf,39865,-1] 2024-11-11T07:24:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:24:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:24:42,519 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:24:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a24bfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:24:42,520 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T07:24:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T07:24:42,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:42,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:42,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:42,528 DEBUG [Thread-2373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,529 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,536 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,537 DEBUG [Thread-2362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,537 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,538 DEBUG [Thread-2362 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,538 DEBUG [Thread-2358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,540 DEBUG [Thread-2375 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,543 DEBUG [Thread-2360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,544 DEBUG [Thread-2364 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:24:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f334e8b0fbdf,39329,1731309728191, seqNum=-1] 2024-11-11T07:24:42,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:24:42,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:24:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5] 2024-11-11T07:24:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,552 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:42,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:24:42,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a5c80f6ee8b1490789f4e50fb4cadc57_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309882552/Put/seqid=0 2024-11-11T07:24:42,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309942574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309942575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309942575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309942577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309942577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742352_1528 (size=12154) 2024-11-11T07:24:42,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:42,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a5c80f6ee8b1490789f4e50fb4cadc57_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a5c80f6ee8b1490789f4e50fb4cadc57_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/965fa37f5ee34b7f84975c649e604fa5, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:42,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/965fa37f5ee34b7f84975c649e604fa5 is 175, key is test_row_0/A:col10/1731309882552/Put/seqid=0 2024-11-11T07:24:42,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742353_1529 (size=30955) 2024-11-11T07:24:42,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/965fa37f5ee34b7f84975c649e604fa5 2024-11-11T07:24:42,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/ae34953d51ea4f0a96a1acc6ed07d10f is 50, key is test_row_0/B:col10/1731309882552/Put/seqid=0 2024-11-11T07:24:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T07:24:42,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742354_1530 (size=12001) 2024-11-11T07:24:42,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/ae34953d51ea4f0a96a1acc6ed07d10f 2024-11-11T07:24:42,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/0bab1344c9ad462e8a018ef4439eb206 is 50, key is test_row_0/C:col10/1731309882552/Put/seqid=0 2024-11-11T07:24:42,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T07:24:42,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:42,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742355_1531 (size=12001) 2024-11-11T07:24:42,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/0bab1344c9ad462e8a018ef4439eb206 2024-11-11T07:24:42,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/965fa37f5ee34b7f84975c649e604fa5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/965fa37f5ee34b7f84975c649e604fa5 2024-11-11T07:24:42,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/965fa37f5ee34b7f84975c649e604fa5, entries=150, sequenceid=16, filesize=30.2 K 2024-11-11T07:24:42,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/ae34953d51ea4f0a96a1acc6ed07d10f as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/ae34953d51ea4f0a96a1acc6ed07d10f 2024-11-11T07:24:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309942692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309942692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309942692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309942692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309942692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/ae34953d51ea4f0a96a1acc6ed07d10f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T07:24:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/0bab1344c9ad462e8a018ef4439eb206 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/0bab1344c9ad462e8a018ef4439eb206 2024-11-11T07:24:42,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/0bab1344c9ad462e8a018ef4439eb206, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T07:24:42,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 022a345c4df8dd612d037e1eac2d4556 in 152ms, sequenceid=16, compaction requested=false 2024-11-11T07:24:42,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:42,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T07:24:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:42,833 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:24:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T07:24:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118cd0ab4f1fef4d26bab4caa0ccdedd42_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309882575/Put/seqid=0 2024-11-11T07:24:42,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742356_1532 (size=12154) 2024-11-11T07:24:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:42,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309942909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309942909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309942911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309942912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:42,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309942913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309943022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309943022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309943022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309943022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309943022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T07:24:43,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309943231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309943231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309943232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309943232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309943232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:43,258 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118cd0ab4f1fef4d26bab4caa0ccdedd42_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118cd0ab4f1fef4d26bab4caa0ccdedd42_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:43,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/a88b877fe8cd46928cabb7e8531b43ce, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:43,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/a88b877fe8cd46928cabb7e8531b43ce is 175, key is test_row_0/A:col10/1731309882575/Put/seqid=0 2024-11-11T07:24:43,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742357_1533 (size=30955) 2024-11-11T07:24:43,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309943542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309943542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309943542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309943542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309943542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:43,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T07:24:43,673 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/a88b877fe8cd46928cabb7e8531b43ce 2024-11-11T07:24:43,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/668e29b1c2c1459aad6a085d185cc3cd is 50, key is test_row_0/B:col10/1731309882575/Put/seqid=0 2024-11-11T07:24:43,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742358_1534 (size=12001) 2024-11-11T07:24:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309944051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309944051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309944052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309944052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309944052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:44,085 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/668e29b1c2c1459aad6a085d185cc3cd 2024-11-11T07:24:44,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/5269da2843bf473db329b516ad8581fd is 50, key is test_row_0/C:col10/1731309882575/Put/seqid=0 2024-11-11T07:24:44,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742359_1535 (size=12001) 2024-11-11T07:24:44,501 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/5269da2843bf473db329b516ad8581fd 2024-11-11T07:24:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/a88b877fe8cd46928cabb7e8531b43ce as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/a88b877fe8cd46928cabb7e8531b43ce 2024-11-11T07:24:44,511 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/a88b877fe8cd46928cabb7e8531b43ce, entries=150, sequenceid=41, filesize=30.2 K 2024-11-11T07:24:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/668e29b1c2c1459aad6a085d185cc3cd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/668e29b1c2c1459aad6a085d185cc3cd 2024-11-11T07:24:44,529 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/668e29b1c2c1459aad6a085d185cc3cd, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T07:24:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/5269da2843bf473db329b516ad8581fd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/5269da2843bf473db329b516ad8581fd 2024-11-11T07:24:44,540 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/5269da2843bf473db329b516ad8581fd, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T07:24:44,541 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 022a345c4df8dd612d037e1eac2d4556 in 1708ms, sequenceid=41, compaction requested=false 2024-11-11T07:24:44,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:44,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:44,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T07:24:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T07:24:44,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T07:24:44,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-11T07:24:44,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.0210 sec 2024-11-11T07:24:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T07:24:44,672 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:44,673 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T07:24:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T07:24:44,675 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:44,676 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:44,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:44,761 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:24:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T07:24:44,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T07:24:44,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:44,830 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:24:44,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b876ff274f24c6f946c3336c28e4c95_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309882912/Put/seqid=0 2024-11-11T07:24:44,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742360_1536 (size=12154) 2024-11-11T07:24:44,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,853 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b876ff274f24c6f946c3336c28e4c95_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b876ff274f24c6f946c3336c28e4c95_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/23ba3eb2e86a4e6594db01f4e4bbef3d, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/23ba3eb2e86a4e6594db01f4e4bbef3d is 175, key is test_row_0/A:col10/1731309882912/Put/seqid=0 2024-11-11T07:24:44,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742361_1537 (size=30955) 2024-11-11T07:24:44,886 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/23ba3eb2e86a4e6594db01f4e4bbef3d 2024-11-11T07:24:44,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/795127a2e0194500a751433aa3e3fcba is 50, key is test_row_0/B:col10/1731309882912/Put/seqid=0 2024-11-11T07:24:44,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742362_1538 (size=12001) 2024-11-11T07:24:44,951 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/795127a2e0194500a751433aa3e3fcba 2024-11-11T07:24:44,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/f8e4593d1ee6497d9a382bfa9af810f8 is 50, key is test_row_0/C:col10/1731309882912/Put/seqid=0 2024-11-11T07:24:44,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742363_1539 (size=12001) 2024-11-11T07:24:44,966 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/f8e4593d1ee6497d9a382bfa9af810f8 2024-11-11T07:24:44,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/23ba3eb2e86a4e6594db01f4e4bbef3d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/23ba3eb2e86a4e6594db01f4e4bbef3d 2024-11-11T07:24:44,977 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/23ba3eb2e86a4e6594db01f4e4bbef3d, entries=150, sequenceid=52, filesize=30.2 K 2024-11-11T07:24:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/795127a2e0194500a751433aa3e3fcba as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/795127a2e0194500a751433aa3e3fcba 2024-11-11T07:24:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/795127a2e0194500a751433aa3e3fcba, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T07:24:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/f8e4593d1ee6497d9a382bfa9af810f8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f8e4593d1ee6497d9a382bfa9af810f8 2024-11-11T07:24:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f8e4593d1ee6497d9a382bfa9af810f8, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T07:24:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,990 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 022a345c4df8dd612d037e1eac2d4556 in 161ms, sequenceid=52, compaction requested=true 2024-11-11T07:24:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T07:24:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T07:24:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T07:24:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T07:24:44,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 316 msec 2024-11-11T07:24:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 321 msec 2024-11-11T07:24:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:44,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:45,105 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:45,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T07:24:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b360321e3cef4f78aca3ada3c532ee70_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309885103/Put/seqid=0 2024-11-11T07:24:45,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742365_1541 (size=24358) 2024-11-11T07:24:45,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309945161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309945164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309945169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309945170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309945171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309945281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309945281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309945281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309945282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309945282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T07:24:45,301 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:45,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T07:24:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T07:24:45,303 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:45,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:45,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T07:24:45,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:45,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:45,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309945492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309945492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309945492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309945492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309945492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,560 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:45,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b360321e3cef4f78aca3ada3c532ee70_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b360321e3cef4f78aca3ada3c532ee70_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:45,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ec56daa719d34aa0a0c3a15a72a7c4d6, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:45,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ec56daa719d34aa0a0c3a15a72a7c4d6 is 175, key is test_row_0/A:col10/1731309885103/Put/seqid=0 2024-11-11T07:24:45,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742364_1540 (size=73995) 2024-11-11T07:24:45,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T07:24:45,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309945801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309945801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309945801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309945802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309945802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:45,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:45,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T07:24:45,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ec56daa719d34aa0a0c3a15a72a7c4d6 2024-11-11T07:24:45,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/3dfceb1954b44bd7b74ca96d684d1b0e is 50, key is test_row_0/B:col10/1731309885103/Put/seqid=0 2024-11-11T07:24:45,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742366_1542 (size=12001) 2024-11-11T07:24:45,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/3dfceb1954b44bd7b74ca96d684d1b0e 2024-11-11T07:24:45,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/9d87a3446a8e402ea5ba8437c299310e is 50, key is test_row_0/C:col10/1731309885103/Put/seqid=0 2024-11-11T07:24:45,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742367_1543 (size=12001) 2024-11-11T07:24:46,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:46,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:46,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309946311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309946312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309946312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309946312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309946312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:46,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:46,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:46,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/9d87a3446a8e402ea5ba8437c299310e 2024-11-11T07:24:46,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ec56daa719d34aa0a0c3a15a72a7c4d6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ec56daa719d34aa0a0c3a15a72a7c4d6 2024-11-11T07:24:46,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ec56daa719d34aa0a0c3a15a72a7c4d6, entries=400, sequenceid=63, filesize=72.3 K 2024-11-11T07:24:46,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/3dfceb1954b44bd7b74ca96d684d1b0e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3dfceb1954b44bd7b74ca96d684d1b0e 2024-11-11T07:24:46,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3dfceb1954b44bd7b74ca96d684d1b0e, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T07:24:46,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/9d87a3446a8e402ea5ba8437c299310e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/9d87a3446a8e402ea5ba8437c299310e 2024-11-11T07:24:46,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/9d87a3446a8e402ea5ba8437c299310e, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T07:24:46,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 022a345c4df8dd612d037e1eac2d4556 in 1310ms, sequenceid=63, compaction requested=true 2024-11-11T07:24:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:46,415 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:46,415 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:46,416 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166860 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:46,416 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:24:46,416 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,417 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/965fa37f5ee34b7f84975c649e604fa5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/a88b877fe8cd46928cabb7e8531b43ce, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/23ba3eb2e86a4e6594db01f4e4bbef3d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ec56daa719d34aa0a0c3a15a72a7c4d6] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=162.9 K 2024-11-11T07:24:46,417 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,417 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/965fa37f5ee34b7f84975c649e604fa5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/a88b877fe8cd46928cabb7e8531b43ce, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/23ba3eb2e86a4e6594db01f4e4bbef3d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ec56daa719d34aa0a0c3a15a72a7c4d6] 2024-11-11T07:24:46,417 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:46,417 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965fa37f5ee34b7f84975c649e604fa5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731309882549 2024-11-11T07:24:46,417 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:24:46,417 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,417 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/ae34953d51ea4f0a96a1acc6ed07d10f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/668e29b1c2c1459aad6a085d185cc3cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/795127a2e0194500a751433aa3e3fcba, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3dfceb1954b44bd7b74ca96d684d1b0e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=46.9 K 2024-11-11T07:24:46,417 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting a88b877fe8cd46928cabb7e8531b43ce, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731309882569 2024-11-11T07:24:46,418 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting ae34953d51ea4f0a96a1acc6ed07d10f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731309882549 2024-11-11T07:24:46,418 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ba3eb2e86a4e6594db01f4e4bbef3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309882905 2024-11-11T07:24:46,418 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 668e29b1c2c1459aad6a085d185cc3cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731309882569 2024-11-11T07:24:46,418 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec56daa719d34aa0a0c3a15a72a7c4d6, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731309885086 2024-11-11T07:24:46,418 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 795127a2e0194500a751433aa3e3fcba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309882905 2024-11-11T07:24:46,419 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dfceb1954b44bd7b74ca96d684d1b0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731309885088 2024-11-11T07:24:46,428 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:46,428 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/3d58b4950f2d4e86a37f9b307a701cd4 is 50, key is test_row_0/B:col10/1731309885103/Put/seqid=0 2024-11-11T07:24:46,433 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T07:24:46,441 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111127c57c2bd0be47329ddf202b44ef5c6d_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:46,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742368_1544 (size=12139) 2024-11-11T07:24:46,445 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111127c57c2bd0be47329ddf202b44ef5c6d_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:46,445 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111127c57c2bd0be47329ddf202b44ef5c6d_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:46,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742369_1545 (size=4469) 2024-11-11T07:24:46,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T07:24:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,527 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T07:24:46,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112591794f90e24766b8568f17718091e8_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309885168/Put/seqid=0 2024-11-11T07:24:46,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742370_1546 (size=12154) 2024-11-11T07:24:46,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:46,550 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112591794f90e24766b8568f17718091e8_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112591794f90e24766b8568f17718091e8_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:46,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/4313fb6b52ed47c6bed787d4be9bc881, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:46,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/4313fb6b52ed47c6bed787d4be9bc881 is 175, key is test_row_0/A:col10/1731309885168/Put/seqid=0 2024-11-11T07:24:46,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742371_1547 (size=30955) 2024-11-11T07:24:46,851 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:46,851 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/c00bba6dd4c140beb6e32112dce686d8 is 175, key is test_row_0/A:col10/1731309885103/Put/seqid=0 2024-11-11T07:24:46,856 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/3d58b4950f2d4e86a37f9b307a701cd4 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3d58b4950f2d4e86a37f9b307a701cd4 2024-11-11T07:24:46,861 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into 3d58b4950f2d4e86a37f9b307a701cd4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:46,861 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:46,861 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=12, startTime=1731309886415; duration=0sec 2024-11-11T07:24:46,861 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:46,861 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:24:46,861 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:46,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742372_1548 (size=31093) 2024-11-11T07:24:46,863 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:46,863 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:24:46,863 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:46,863 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/0bab1344c9ad462e8a018ef4439eb206, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/5269da2843bf473db329b516ad8581fd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f8e4593d1ee6497d9a382bfa9af810f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/9d87a3446a8e402ea5ba8437c299310e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=46.9 K 2024-11-11T07:24:46,863 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bab1344c9ad462e8a018ef4439eb206, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731309882549 2024-11-11T07:24:46,863 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 5269da2843bf473db329b516ad8581fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731309882569 2024-11-11T07:24:46,864 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e4593d1ee6497d9a382bfa9af810f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731309882905 2024-11-11T07:24:46,864 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d87a3446a8e402ea5ba8437c299310e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731309885088 2024-11-11T07:24:46,870 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:46,871 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/09730a16a3ff43cdb4f1de2ea3728183 is 50, key is test_row_0/C:col10/1731309885103/Put/seqid=0 2024-11-11T07:24:46,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742373_1549 (size=12139) 2024-11-11T07:24:46,882 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/09730a16a3ff43cdb4f1de2ea3728183 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/09730a16a3ff43cdb4f1de2ea3728183 2024-11-11T07:24:46,886 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into 09730a16a3ff43cdb4f1de2ea3728183(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:46,886 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:46,886 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=12, startTime=1731309886415; duration=0sec 2024-11-11T07:24:46,886 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:46,886 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:24:46,956 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/4313fb6b52ed47c6bed787d4be9bc881 2024-11-11T07:24:46,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/f1f91ce995064c59a4d55510d01aead1 is 50, key is test_row_0/B:col10/1731309885168/Put/seqid=0 2024-11-11T07:24:46,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742374_1550 (size=12001) 2024-11-11T07:24:47,266 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/c00bba6dd4c140beb6e32112dce686d8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c00bba6dd4c140beb6e32112dce686d8 2024-11-11T07:24:47,270 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into c00bba6dd4c140beb6e32112dce686d8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:47,270 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:47,270 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=12, startTime=1731309886415; duration=0sec 2024-11-11T07:24:47,270 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:47,270 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:24:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:47,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:47,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309947332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309947333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,335 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309947341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309947341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309947345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,369 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/f1f91ce995064c59a4d55510d01aead1 2024-11-11T07:24:47,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/1870f83a0e714786b3271640f9c3548e is 50, key is test_row_0/C:col10/1731309885168/Put/seqid=0 2024-11-11T07:24:47,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742375_1551 (size=12001) 2024-11-11T07:24:47,387 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/1870f83a0e714786b3271640f9c3548e 2024-11-11T07:24:47,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/4313fb6b52ed47c6bed787d4be9bc881 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/4313fb6b52ed47c6bed787d4be9bc881 2024-11-11T07:24:47,394 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/4313fb6b52ed47c6bed787d4be9bc881, entries=150, sequenceid=89, filesize=30.2 K 2024-11-11T07:24:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/f1f91ce995064c59a4d55510d01aead1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/f1f91ce995064c59a4d55510d01aead1 2024-11-11T07:24:47,398 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/f1f91ce995064c59a4d55510d01aead1, entries=150, sequenceid=89, filesize=11.7 K 2024-11-11T07:24:47,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/1870f83a0e714786b3271640f9c3548e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1870f83a0e714786b3271640f9c3548e 2024-11-11T07:24:47,404 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1870f83a0e714786b3271640f9c3548e, entries=150, sequenceid=89, filesize=11.7 K 2024-11-11T07:24:47,405 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 022a345c4df8dd612d037e1eac2d4556 in 878ms, sequenceid=89, compaction requested=false 2024-11-11T07:24:47,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:47,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:47,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T07:24:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T07:24:47,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T07:24:47,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1020 sec 2024-11-11T07:24:47,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.1070 sec 2024-11-11T07:24:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:47,442 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:47,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:24:47,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111e5457c1ef4b478ab12dd284b8175611_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309887441/Put/seqid=0 2024-11-11T07:24:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T07:24:47,451 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:47,452 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T07:24:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:24:47,454 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:47,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742376_1552 (size=14594) 2024-11-11T07:24:47,455 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:47,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:47,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:47,460 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111e5457c1ef4b478ab12dd284b8175611_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e5457c1ef4b478ab12dd284b8175611_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:47,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/c1be6180b4444ed7888c60c20f172d88, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:47,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/c1be6180b4444ed7888c60c20f172d88 is 175, key is test_row_0/A:col10/1731309887441/Put/seqid=0 2024-11-11T07:24:47,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742377_1553 (size=39549) 2024-11-11T07:24:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309947480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309947486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309947486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309947487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:24:47,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309947592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309947602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309947602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309947602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T07:24:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:47,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T07:24:47,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:47,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:24:47,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309947802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309947812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:47,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/c1be6180b4444ed7888c60c20f172d88 2024-11-11T07:24:47,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8fd57213cfbc4b2081143246b8ec710b is 50, key is test_row_0/B:col10/1731309887441/Put/seqid=0 2024-11-11T07:24:47,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742378_1554 (size=12001) 2024-11-11T07:24:47,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8fd57213cfbc4b2081143246b8ec710b 2024-11-11T07:24:47,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/a2c7b63503e640ec8d6dd2a6f7e83df5 is 50, key is test_row_0/C:col10/1731309887441/Put/seqid=0 2024-11-11T07:24:47,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742379_1555 (size=12001) 2024-11-11T07:24:47,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T07:24:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:47,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:48,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T07:24:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:48,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:24:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309948112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309948122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309948122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309948122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T07:24:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:48,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/a2c7b63503e640ec8d6dd2a6f7e83df5 2024-11-11T07:24:48,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/c1be6180b4444ed7888c60c20f172d88 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c1be6180b4444ed7888c60c20f172d88 2024-11-11T07:24:48,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c1be6180b4444ed7888c60c20f172d88, entries=200, sequenceid=104, filesize=38.6 K 2024-11-11T07:24:48,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8fd57213cfbc4b2081143246b8ec710b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8fd57213cfbc4b2081143246b8ec710b 2024-11-11T07:24:48,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8fd57213cfbc4b2081143246b8ec710b, entries=150, sequenceid=104, filesize=11.7 K 2024-11-11T07:24:48,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/a2c7b63503e640ec8d6dd2a6f7e83df5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/a2c7b63503e640ec8d6dd2a6f7e83df5 2024-11-11T07:24:48,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/a2c7b63503e640ec8d6dd2a6f7e83df5, entries=150, sequenceid=104, filesize=11.7 K 2024-11-11T07:24:48,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 022a345c4df8dd612d037e1eac2d4556 in 878ms, sequenceid=104, compaction requested=true 2024-11-11T07:24:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:48,321 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:24:48,321 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:48,322 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:48,322 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:48,322 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:24:48,322 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:24:48,322 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,322 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,322 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3d58b4950f2d4e86a37f9b307a701cd4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/f1f91ce995064c59a4d55510d01aead1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8fd57213cfbc4b2081143246b8ec710b] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=35.3 K 2024-11-11T07:24:48,322 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c00bba6dd4c140beb6e32112dce686d8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/4313fb6b52ed47c6bed787d4be9bc881, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c1be6180b4444ed7888c60c20f172d88] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=99.2 K 2024-11-11T07:24:48,322 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,322 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c00bba6dd4c140beb6e32112dce686d8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/4313fb6b52ed47c6bed787d4be9bc881, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c1be6180b4444ed7888c60c20f172d88] 2024-11-11T07:24:48,323 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d58b4950f2d4e86a37f9b307a701cd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731309885088 2024-11-11T07:24:48,323 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c00bba6dd4c140beb6e32112dce686d8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731309885088 2024-11-11T07:24:48,323 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f91ce995064c59a4d55510d01aead1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731309885153 2024-11-11T07:24:48,323 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4313fb6b52ed47c6bed787d4be9bc881, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731309885153 2024-11-11T07:24:48,324 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd57213cfbc4b2081143246b8ec710b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731309887326 2024-11-11T07:24:48,324 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1be6180b4444ed7888c60c20f172d88, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731309887326 2024-11-11T07:24:48,338 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:48,340 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:48,341 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/673a4bd69a6d4cfeae5b98cfc5fb99f8 is 50, key is test_row_0/B:col10/1731309887441/Put/seqid=0 2024-11-11T07:24:48,342 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c2d4607342aa45348b8b935fc2023ca3_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:48,344 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c2d4607342aa45348b8b935fc2023ca3_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:48,344 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c2d4607342aa45348b8b935fc2023ca3_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:48,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742380_1556 (size=12241) 2024-11-11T07:24:48,350 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/673a4bd69a6d4cfeae5b98cfc5fb99f8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/673a4bd69a6d4cfeae5b98cfc5fb99f8 2024-11-11T07:24:48,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742381_1557 (size=4469) 2024-11-11T07:24:48,352 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#474 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:48,352 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/73ecb46ea25945768dd19c15d3ceda04 is 175, key is test_row_0/A:col10/1731309887441/Put/seqid=0 2024-11-11T07:24:48,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742382_1558 (size=31195) 2024-11-11T07:24:48,356 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into 673a4bd69a6d4cfeae5b98cfc5fb99f8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:48,356 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:48,356 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=13, startTime=1731309888321; duration=0sec 2024-11-11T07:24:48,356 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:48,356 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:24:48,356 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:48,359 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:48,360 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:24:48,360 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,360 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/09730a16a3ff43cdb4f1de2ea3728183, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1870f83a0e714786b3271640f9c3548e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/a2c7b63503e640ec8d6dd2a6f7e83df5] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=35.3 K 2024-11-11T07:24:48,360 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 09730a16a3ff43cdb4f1de2ea3728183, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731309885088 2024-11-11T07:24:48,360 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1870f83a0e714786b3271640f9c3548e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731309885153 2024-11-11T07:24:48,361 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c7b63503e640ec8d6dd2a6f7e83df5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731309887326 2024-11-11T07:24:48,363 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/73ecb46ea25945768dd19c15d3ceda04 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/73ecb46ea25945768dd19c15d3ceda04 2024-11-11T07:24:48,370 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into 73ecb46ea25945768dd19c15d3ceda04(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:48,370 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:48,370 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=13, startTime=1731309888321; duration=0sec 2024-11-11T07:24:48,370 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:48,370 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:24:48,373 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:48,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T07:24:48,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:48,373 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/b4440c86d3db404aa99230a39e8707d6 is 50, key is test_row_0/C:col10/1731309887441/Put/seqid=0 2024-11-11T07:24:48,373 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T07:24:48,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742383_1559 (size=12241) 2024-11-11T07:24:48,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dc1eb48deb514be1b1f2ca57c0ace1cd_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309887484/Put/seqid=0 2024-11-11T07:24:48,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742384_1560 (size=12154) 2024-11-11T07:24:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:24:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:48,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:48,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309948637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309948638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309948639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309948648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309948752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309948752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309948752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309948761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,807 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/b4440c86d3db404aa99230a39e8707d6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b4440c86d3db404aa99230a39e8707d6 2024-11-11T07:24:48,811 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into b4440c86d3db404aa99230a39e8707d6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:48,811 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:48,811 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=13, startTime=1731309888321; duration=0sec 2024-11-11T07:24:48,811 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:48,812 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:24:48,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:48,823 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dc1eb48deb514be1b1f2ca57c0ace1cd_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc1eb48deb514be1b1f2ca57c0ace1cd_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:48,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d7651f92d7f48b5933b8040feb196a1, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:48,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d7651f92d7f48b5933b8040feb196a1 is 175, key is test_row_0/A:col10/1731309887484/Put/seqid=0 2024-11-11T07:24:48,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742385_1561 (size=30955) 2024-11-11T07:24:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309948961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309948961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309948961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:48,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309948972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,228 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d7651f92d7f48b5933b8040feb196a1 2024-11-11T07:24:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/b01c10cb59ea42d0b2c26438cb3bc5c1 is 50, key is test_row_0/B:col10/1731309887484/Put/seqid=0 2024-11-11T07:24:49,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742386_1562 (size=12001) 2024-11-11T07:24:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309949272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309949272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309949272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309949281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309949361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,365 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:24:49,639 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/b01c10cb59ea42d0b2c26438cb3bc5c1 2024-11-11T07:24:49,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/56497654b389405c8ddbb2f18deeb8fe is 50, key is test_row_0/C:col10/1731309887484/Put/seqid=0 2024-11-11T07:24:49,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742387_1563 (size=12001) 2024-11-11T07:24:49,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309949781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309949781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309949781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:49,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309949791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,050 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/56497654b389405c8ddbb2f18deeb8fe 2024-11-11T07:24:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d7651f92d7f48b5933b8040feb196a1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d7651f92d7f48b5933b8040feb196a1 2024-11-11T07:24:50,060 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d7651f92d7f48b5933b8040feb196a1, entries=150, sequenceid=131, filesize=30.2 K 2024-11-11T07:24:50,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/b01c10cb59ea42d0b2c26438cb3bc5c1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b01c10cb59ea42d0b2c26438cb3bc5c1 2024-11-11T07:24:50,064 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b01c10cb59ea42d0b2c26438cb3bc5c1, entries=150, sequenceid=131, filesize=11.7 K 2024-11-11T07:24:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/56497654b389405c8ddbb2f18deeb8fe as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/56497654b389405c8ddbb2f18deeb8fe 2024-11-11T07:24:50,068 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/56497654b389405c8ddbb2f18deeb8fe, entries=150, sequenceid=131, filesize=11.7 K 2024-11-11T07:24:50,069 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 022a345c4df8dd612d037e1eac2d4556 in 1696ms, sequenceid=131, compaction requested=false 2024-11-11T07:24:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T07:24:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T07:24:50,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T07:24:50,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6160 sec 2024-11-11T07:24:50,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.6220 sec 2024-11-11T07:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:50,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:50,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:24:50,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b94c7e4d72e49fb97a5bedb7bf9be17_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309890792/Put/seqid=0 2024-11-11T07:24:50,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742388_1564 (size=12304) 2024-11-11T07:24:50,805 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:50,811 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b94c7e4d72e49fb97a5bedb7bf9be17_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b94c7e4d72e49fb97a5bedb7bf9be17_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:50,811 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be4135f1eb8842c99ab255e01303f718, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:50,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be4135f1eb8842c99ab255e01303f718 is 175, key is test_row_0/A:col10/1731309890792/Put/seqid=0 2024-11-11T07:24:50,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742389_1565 (size=31105) 2024-11-11T07:24:50,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309950823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309950824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309950825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309950826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309950931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309950931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309950932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:50,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309950942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309951141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309951141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309951142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309951152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,216 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be4135f1eb8842c99ab255e01303f718 2024-11-11T07:24:51,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/2eec20061514410290c0cb4dff2b4e23 is 50, key is test_row_0/B:col10/1731309890792/Put/seqid=0 2024-11-11T07:24:51,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742390_1566 (size=12151) 2024-11-11T07:24:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309951451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309951452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309951452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309951461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:24:51,612 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:51,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T07:24:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T07:24:51,614 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:51,615 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:51,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:51,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/2eec20061514410290c0cb4dff2b4e23 2024-11-11T07:24:51,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/de701c7ede1a43c3b91f4e7fe8fcb451 is 50, key is test_row_0/C:col10/1731309890792/Put/seqid=0 2024-11-11T07:24:51,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742391_1567 (size=12151) 2024-11-11T07:24:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T07:24:51,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T07:24:51,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:51,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:51,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:51,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:51,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:51,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T07:24:51,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:51,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T07:24:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309951961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309951961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309951962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309951972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:52,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/de701c7ede1a43c3b91f4e7fe8fcb451 2024-11-11T07:24:52,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be4135f1eb8842c99ab255e01303f718 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be4135f1eb8842c99ab255e01303f718 2024-11-11T07:24:52,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be4135f1eb8842c99ab255e01303f718, entries=150, sequenceid=144, filesize=30.4 K 2024-11-11T07:24:52,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/2eec20061514410290c0cb4dff2b4e23 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/2eec20061514410290c0cb4dff2b4e23 2024-11-11T07:24:52,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/2eec20061514410290c0cb4dff2b4e23, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T07:24:52,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/de701c7ede1a43c3b91f4e7fe8fcb451 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/de701c7ede1a43c3b91f4e7fe8fcb451 2024-11-11T07:24:52,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/de701c7ede1a43c3b91f4e7fe8fcb451, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T07:24:52,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 022a345c4df8dd612d037e1eac2d4556 in 1269ms, sequenceid=144, compaction requested=true 2024-11-11T07:24:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:52,064 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:52,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:52,064 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:52,065 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:52,065 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93255 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:52,065 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:24:52,065 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:24:52,065 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,065 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,065 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/73ecb46ea25945768dd19c15d3ceda04, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d7651f92d7f48b5933b8040feb196a1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be4135f1eb8842c99ab255e01303f718] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=91.1 K 2024-11-11T07:24:52,065 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/673a4bd69a6d4cfeae5b98cfc5fb99f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b01c10cb59ea42d0b2c26438cb3bc5c1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/2eec20061514410290c0cb4dff2b4e23] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=35.5 K 2024-11-11T07:24:52,065 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,065 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/73ecb46ea25945768dd19c15d3ceda04, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d7651f92d7f48b5933b8040feb196a1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be4135f1eb8842c99ab255e01303f718] 2024-11-11T07:24:52,065 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 673a4bd69a6d4cfeae5b98cfc5fb99f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731309887326 2024-11-11T07:24:52,065 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ecb46ea25945768dd19c15d3ceda04, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731309887326 2024-11-11T07:24:52,066 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b01c10cb59ea42d0b2c26438cb3bc5c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309887471 2024-11-11T07:24:52,066 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eec20061514410290c0cb4dff2b4e23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731309888635 2024-11-11T07:24:52,066 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7651f92d7f48b5933b8040feb196a1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309887471 2024-11-11T07:24:52,066 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting be4135f1eb8842c99ab255e01303f718, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731309888635 2024-11-11T07:24:52,071 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#482 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:52,072 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/07095ec554b6416890759945d782d354 is 50, key is test_row_0/B:col10/1731309890792/Put/seqid=0 2024-11-11T07:24:52,073 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:52,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T07:24:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,074 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:24:52,075 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411119d327e78589146c4b5d47398fb9d7372_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:52,077 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411119d327e78589146c4b5d47398fb9d7372_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:52,077 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119d327e78589146c4b5d47398fb9d7372_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:52,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742392_1568 (size=12493) 2024-11-11T07:24:52,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742393_1569 (size=4469) 2024-11-11T07:24:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f4c1d8a7a8f3490bb65015f72a358b4e_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309890822/Put/seqid=0 2024-11-11T07:24:52,091 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#483 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:52,092 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/5ddc578ef87e4ad2b23cdfc81ae854df is 175, key is test_row_0/A:col10/1731309890792/Put/seqid=0 2024-11-11T07:24:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742394_1570 (size=12304) 2024-11-11T07:24:52,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742395_1571 (size=31447) 2024-11-11T07:24:52,100 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f4c1d8a7a8f3490bb65015f72a358b4e_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4c1d8a7a8f3490bb65015f72a358b4e_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:52,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/356526b0075344dda944a978824b21f7, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:52,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/356526b0075344dda944a978824b21f7 is 175, key is test_row_0/A:col10/1731309890822/Put/seqid=0 2024-11-11T07:24:52,106 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/5ddc578ef87e4ad2b23cdfc81ae854df as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/5ddc578ef87e4ad2b23cdfc81ae854df 2024-11-11T07:24:52,111 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into 5ddc578ef87e4ad2b23cdfc81ae854df(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:52,111 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:52,111 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=13, startTime=1731309892064; duration=0sec 2024-11-11T07:24:52,111 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:52,111 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:24:52,111 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:52,112 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:52,112 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:24:52,112 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,112 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b4440c86d3db404aa99230a39e8707d6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/56497654b389405c8ddbb2f18deeb8fe, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/de701c7ede1a43c3b91f4e7fe8fcb451] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=35.5 K 2024-11-11T07:24:52,112 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4440c86d3db404aa99230a39e8707d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731309887326 2024-11-11T07:24:52,112 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56497654b389405c8ddbb2f18deeb8fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731309887471 2024-11-11T07:24:52,114 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting de701c7ede1a43c3b91f4e7fe8fcb451, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731309888635 2024-11-11T07:24:52,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742396_1572 (size=31105) 2024-11-11T07:24:52,116 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/356526b0075344dda944a978824b21f7 2024-11-11T07:24:52,120 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:52,121 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/d6216f8810704c96a53aa6ad068eabcf is 50, key is test_row_0/C:col10/1731309890792/Put/seqid=0 2024-11-11T07:24:52,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/823f4c86c11f4ab6b2d9e314fd13ca67 is 50, key is test_row_0/B:col10/1731309890822/Put/seqid=0 2024-11-11T07:24:52,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742398_1574 (size=12151) 2024-11-11T07:24:52,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742397_1573 (size=12493) 2024-11-11T07:24:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T07:24:52,482 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/07095ec554b6416890759945d782d354 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/07095ec554b6416890759945d782d354 2024-11-11T07:24:52,487 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into 07095ec554b6416890759945d782d354(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:52,487 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:52,487 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=13, startTime=1731309892064; duration=0sec 2024-11-11T07:24:52,487 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:52,487 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:24:52,528 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/823f4c86c11f4ab6b2d9e314fd13ca67 2024-11-11T07:24:52,533 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/d6216f8810704c96a53aa6ad068eabcf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d6216f8810704c96a53aa6ad068eabcf 2024-11-11T07:24:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/bc186568e19e448eb4d17ad3276346ff is 50, key is test_row_0/C:col10/1731309890822/Put/seqid=0 2024-11-11T07:24:52,537 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into d6216f8810704c96a53aa6ad068eabcf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:52,537 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:52,537 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=13, startTime=1731309892064; duration=0sec 2024-11-11T07:24:52,537 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:52,537 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:24:52,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742399_1575 (size=12151) 2024-11-11T07:24:52,540 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/bc186568e19e448eb4d17ad3276346ff 2024-11-11T07:24:52,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/356526b0075344dda944a978824b21f7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/356526b0075344dda944a978824b21f7 2024-11-11T07:24:52,547 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/356526b0075344dda944a978824b21f7, entries=150, sequenceid=168, filesize=30.4 K 2024-11-11T07:24:52,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/823f4c86c11f4ab6b2d9e314fd13ca67 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/823f4c86c11f4ab6b2d9e314fd13ca67 2024-11-11T07:24:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,550 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/823f4c86c11f4ab6b2d9e314fd13ca67, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T07:24:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/bc186568e19e448eb4d17ad3276346ff as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/bc186568e19e448eb4d17ad3276346ff 2024-11-11T07:24:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,554 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/bc186568e19e448eb4d17ad3276346ff, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T07:24:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,555 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 022a345c4df8dd612d037e1eac2d4556 in 481ms, sequenceid=168, compaction requested=false 2024-11-11T07:24:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T07:24:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T07:24:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T07:24:52,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 942 msec 2024-11-11T07:24:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 947 msec 2024-11-11T07:24:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T07:24:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,751 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,752 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-11T07:24:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T07:24:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,754 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,755 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:52,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T07:24:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T07:24:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:52,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:52,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-11T07:24:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-11T07:24:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T07:24:52,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-11T07:24:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 161 msec 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,007 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:53,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T07:24:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111163435aea945f47c2ac5967cb8815c3eb_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309892998/Put/seqid=0 2024-11-11T07:24:53,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742401_1577 (size=29738) 2024-11-11T07:24:53,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309953035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309953036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309953039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309953040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T07:24:53,072 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:53,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-11T07:24:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T07:24:53,074 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:53,075 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:53,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:53,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309953141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309953152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309953152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309953152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T07:24:53,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309953351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309953361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309953362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309953362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T07:24:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309953401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,404 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,420 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:53,424 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111163435aea945f47c2ac5967cb8815c3eb_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163435aea945f47c2ac5967cb8815c3eb_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:53,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/1fd28cbdbfb84a7e9aa375f4730d7ce8, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:53,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/1fd28cbdbfb84a7e9aa375f4730d7ce8 is 175, key is test_row_0/A:col10/1731309892998/Put/seqid=0 2024-11-11T07:24:53,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742400_1576 (size=91679) 2024-11-11T07:24:53,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/1fd28cbdbfb84a7e9aa375f4730d7ce8 2024-11-11T07:24:53,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e is 50, key is test_row_0/B:col10/1731309892998/Put/seqid=0 2024-11-11T07:24:53,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742402_1578 (size=12151) 2024-11-11T07:24:53,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309953661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309953671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309953672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309953672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:53,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T07:24:53,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e 2024-11-11T07:24:53,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/c6c1eef571a445d8aec1a6b0abeeb6e6 is 50, key is test_row_0/C:col10/1731309892998/Put/seqid=0 2024-11-11T07:24:53,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742403_1579 (size=12151) 2024-11-11T07:24:53,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:53,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:54,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:54,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:54,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309954171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309954182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309954182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309954182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:54,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T07:24:54,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/c6c1eef571a445d8aec1a6b0abeeb6e6 2024-11-11T07:24:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/1fd28cbdbfb84a7e9aa375f4730d7ce8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/1fd28cbdbfb84a7e9aa375f4730d7ce8 2024-11-11T07:24:54,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/1fd28cbdbfb84a7e9aa375f4730d7ce8, entries=500, sequenceid=185, filesize=89.5 K 2024-11-11T07:24:54,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e 2024-11-11T07:24:54,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T07:24:54,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/c6c1eef571a445d8aec1a6b0abeeb6e6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/c6c1eef571a445d8aec1a6b0abeeb6e6 2024-11-11T07:24:54,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/c6c1eef571a445d8aec1a6b0abeeb6e6, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T07:24:54,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 022a345c4df8dd612d037e1eac2d4556 in 1261ms, sequenceid=185, compaction requested=true 2024-11-11T07:24:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:54,268 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:54,268 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:54,269 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 154231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:54,269 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:54,269 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:24:54,269 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:24:54,269 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:54,269 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:54,269 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/5ddc578ef87e4ad2b23cdfc81ae854df, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/356526b0075344dda944a978824b21f7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/1fd28cbdbfb84a7e9aa375f4730d7ce8] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=150.6 K 2024-11-11T07:24:54,269 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/07095ec554b6416890759945d782d354, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/823f4c86c11f4ab6b2d9e314fd13ca67, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=35.9 K 2024-11-11T07:24:54,269 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:54,269 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/5ddc578ef87e4ad2b23cdfc81ae854df, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/356526b0075344dda944a978824b21f7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/1fd28cbdbfb84a7e9aa375f4730d7ce8] 2024-11-11T07:24:54,271 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 07095ec554b6416890759945d782d354, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731309888635 2024-11-11T07:24:54,271 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ddc578ef87e4ad2b23cdfc81ae854df, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731309888635 2024-11-11T07:24:54,272 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 823f4c86c11f4ab6b2d9e314fd13ca67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731309890822 2024-11-11T07:24:54,272 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356526b0075344dda944a978824b21f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731309890822 2024-11-11T07:24:54,276 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fd28cbdbfb84a7e9aa375f4730d7ce8, keycount=500, bloomtype=ROW, size=89.5 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731309892988 2024-11-11T07:24:54,276 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e8f73d0fd4c4b29aa23e7f0678d4f3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731309892998 2024-11-11T07:24:54,297 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:54,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T07:24:54,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:54,298 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T07:24:54,305 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110247267f8c974d9f8c0119b9784953cf_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:54,306 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:54,307 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/5293bede63664dcf82eee611a6c6d0cd is 50, key is test_row_0/B:col10/1731309892998/Put/seqid=0 2024-11-11T07:24:54,308 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110247267f8c974d9f8c0119b9784953cf_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:54,308 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110247267f8c974d9f8c0119b9784953cf_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:54,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742404_1580 (size=12595) 2024-11-11T07:24:54,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d83131656ae94377acb14b0ba860532b_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309893035/Put/seqid=0 2024-11-11T07:24:54,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742405_1581 (size=4469) 2024-11-11T07:24:54,356 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/5293bede63664dcf82eee611a6c6d0cd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/5293bede63664dcf82eee611a6c6d0cd 2024-11-11T07:24:54,357 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#491 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:54,357 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/b58ad82973be456a8ac94676bd9c9165 is 175, key is test_row_0/A:col10/1731309892998/Put/seqid=0 2024-11-11T07:24:54,361 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into 5293bede63664dcf82eee611a6c6d0cd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:54,361 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:54,361 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=13, startTime=1731309894268; duration=0sec 2024-11-11T07:24:54,361 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:54,361 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:24:54,362 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:54,363 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:54,363 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:24:54,363 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:54,363 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d6216f8810704c96a53aa6ad068eabcf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/bc186568e19e448eb4d17ad3276346ff, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/c6c1eef571a445d8aec1a6b0abeeb6e6] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=35.9 K 2024-11-11T07:24:54,364 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting d6216f8810704c96a53aa6ad068eabcf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731309888635 2024-11-11T07:24:54,364 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting bc186568e19e448eb4d17ad3276346ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731309890822 2024-11-11T07:24:54,365 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c1eef571a445d8aec1a6b0abeeb6e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731309892998 2024-11-11T07:24:54,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742407_1583 (size=31549) 2024-11-11T07:24:54,379 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:54,380 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/1e11cc4b141a43faafd51571a40b24d5 is 50, key is test_row_0/C:col10/1731309892998/Put/seqid=0 2024-11-11T07:24:54,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742406_1582 (size=12304) 2024-11-11T07:24:54,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742408_1584 (size=12595) 2024-11-11T07:24:54,783 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/b58ad82973be456a8ac94676bd9c9165 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/b58ad82973be456a8ac94676bd9c9165 2024-11-11T07:24:54,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:54,788 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into b58ad82973be456a8ac94676bd9c9165(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:54,788 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:54,788 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=13, startTime=1731309894268; duration=0sec 2024-11-11T07:24:54,788 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:54,788 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:24:54,795 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d83131656ae94377acb14b0ba860532b_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d83131656ae94377acb14b0ba860532b_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:54,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/260c8d1bd3b04500a4cec219a563fd15, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:54,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/260c8d1bd3b04500a4cec219a563fd15 is 175, key is test_row_0/A:col10/1731309893035/Put/seqid=0 2024-11-11T07:24:54,801 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/1e11cc4b141a43faafd51571a40b24d5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1e11cc4b141a43faafd51571a40b24d5 2024-11-11T07:24:54,805 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into 1e11cc4b141a43faafd51571a40b24d5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:54,805 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:54,805 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=13, startTime=1731309894268; duration=0sec 2024-11-11T07:24:54,805 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:54,805 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:24:54,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742409_1585 (size=31105) 2024-11-11T07:24:54,811 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/260c8d1bd3b04500a4cec219a563fd15 2024-11-11T07:24:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/abc1b6750d514a61bee302867ccc7d72 is 50, key is test_row_0/B:col10/1731309893035/Put/seqid=0 2024-11-11T07:24:54,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742410_1586 (size=12151) 2024-11-11T07:24:54,822 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/abc1b6750d514a61bee302867ccc7d72 2024-11-11T07:24:54,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/710ef110e543408e912ea328330923da is 50, key is test_row_0/C:col10/1731309893035/Put/seqid=0 2024-11-11T07:24:54,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742411_1587 (size=12151) 2024-11-11T07:24:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:55,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:55,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309955201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309955201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309955205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,206 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:55,206 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309955206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T07:24:55,233 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/710ef110e543408e912ea328330923da 2024-11-11T07:24:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/260c8d1bd3b04500a4cec219a563fd15 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/260c8d1bd3b04500a4cec219a563fd15 2024-11-11T07:24:55,242 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/260c8d1bd3b04500a4cec219a563fd15, entries=150, sequenceid=209, filesize=30.4 K 2024-11-11T07:24:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/abc1b6750d514a61bee302867ccc7d72 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/abc1b6750d514a61bee302867ccc7d72 2024-11-11T07:24:55,246 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/abc1b6750d514a61bee302867ccc7d72, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:24:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/710ef110e543408e912ea328330923da as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/710ef110e543408e912ea328330923da 2024-11-11T07:24:55,250 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/710ef110e543408e912ea328330923da, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T07:24:55,251 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 022a345c4df8dd612d037e1eac2d4556 in 953ms, sequenceid=209, compaction requested=false 2024-11-11T07:24:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-11T07:24:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-11T07:24:55,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T07:24:55,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1770 sec 2024-11-11T07:24:55,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1820 sec 2024-11-11T07:24:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:55,314 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:55,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:24:55,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fdce3b081d2d44daabba9e97f8f24fc1_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309895204/Put/seqid=0 2024-11-11T07:24:55,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742412_1588 (size=14794) 2024-11-11T07:24:55,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:55,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fdce3b081d2d44daabba9e97f8f24fc1_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fdce3b081d2d44daabba9e97f8f24fc1_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:55,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/6d2f03eb0b9a4d5eb4e1341ed8682865, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:55,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/6d2f03eb0b9a4d5eb4e1341ed8682865 is 175, key is test_row_0/A:col10/1731309895204/Put/seqid=0 2024-11-11T07:24:55,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742413_1589 (size=39749) 2024-11-11T07:24:55,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/6d2f03eb0b9a4d5eb4e1341ed8682865 2024-11-11T07:24:55,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/bb880f22bee94c58a2ff5953cc3c46df is 50, key is test_row_0/B:col10/1731309895204/Put/seqid=0 2024-11-11T07:24:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742414_1590 (size=12151) 2024-11-11T07:24:55,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309955369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309955375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309955485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309955486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309955691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309955702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:55,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/bb880f22bee94c58a2ff5953cc3c46df 2024-11-11T07:24:55,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/3dc3eb278db14f3c9ac8dcc900b81aae is 50, key is test_row_0/C:col10/1731309895204/Put/seqid=0 2024-11-11T07:24:55,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742415_1591 (size=12151) 2024-11-11T07:24:56,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309956001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309956011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/3dc3eb278db14f3c9ac8dcc900b81aae 2024-11-11T07:24:56,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/6d2f03eb0b9a4d5eb4e1341ed8682865 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/6d2f03eb0b9a4d5eb4e1341ed8682865 2024-11-11T07:24:56,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/6d2f03eb0b9a4d5eb4e1341ed8682865, entries=200, sequenceid=226, filesize=38.8 K 2024-11-11T07:24:56,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/bb880f22bee94c58a2ff5953cc3c46df as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/bb880f22bee94c58a2ff5953cc3c46df 2024-11-11T07:24:56,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/bb880f22bee94c58a2ff5953cc3c46df, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T07:24:56,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/3dc3eb278db14f3c9ac8dcc900b81aae as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/3dc3eb278db14f3c9ac8dcc900b81aae 2024-11-11T07:24:56,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/3dc3eb278db14f3c9ac8dcc900b81aae, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T07:24:56,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 022a345c4df8dd612d037e1eac2d4556 in 883ms, sequenceid=226, compaction requested=true 2024-11-11T07:24:56,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:56,199 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:56,200 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:56,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:56,201 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:56,201 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:24:56,201 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:56,201 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/b58ad82973be456a8ac94676bd9c9165, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/260c8d1bd3b04500a4cec219a563fd15, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/6d2f03eb0b9a4d5eb4e1341ed8682865] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=100.0 K 2024-11-11T07:24:56,201 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:56,201 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/b58ad82973be456a8ac94676bd9c9165, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/260c8d1bd3b04500a4cec219a563fd15, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/6d2f03eb0b9a4d5eb4e1341ed8682865] 2024-11-11T07:24:56,202 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b58ad82973be456a8ac94676bd9c9165, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731309892998 2024-11-11T07:24:56,203 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:56,203 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260c8d1bd3b04500a4cec219a563fd15, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309893035 2024-11-11T07:24:56,203 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:24:56,203 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:56,203 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/5293bede63664dcf82eee611a6c6d0cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/abc1b6750d514a61bee302867ccc7d72, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/bb880f22bee94c58a2ff5953cc3c46df] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=36.0 K 2024-11-11T07:24:56,203 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2f03eb0b9a4d5eb4e1341ed8682865, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731309895204 2024-11-11T07:24:56,204 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 5293bede63664dcf82eee611a6c6d0cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731309892998 2024-11-11T07:24:56,204 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting abc1b6750d514a61bee302867ccc7d72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309893035 2024-11-11T07:24:56,205 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting bb880f22bee94c58a2ff5953cc3c46df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731309895204 2024-11-11T07:24:56,213 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:56,214 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/e4fd2d1216424546a0f5fcea6c4fe6e5 is 50, key is test_row_0/B:col10/1731309895204/Put/seqid=0 2024-11-11T07:24:56,217 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:56,227 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117c24dea640504d268672bc0549dc5eb1_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:56,228 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117c24dea640504d268672bc0549dc5eb1_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:56,228 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117c24dea640504d268672bc0549dc5eb1_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:56,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742416_1592 (size=12697) 2024-11-11T07:24:56,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742417_1593 (size=4469) 2024-11-11T07:24:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:56,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:56,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:24:56,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d11177518ca432e8ae12bcddf14dc9d_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309895368/Put/seqid=0 2024-11-11T07:24:56,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742418_1594 (size=14794) 2024-11-11T07:24:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309956553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309956554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,645 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:56,646 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/8a1836b4088b4717900a1d193435f344 is 175, key is test_row_0/A:col10/1731309895204/Put/seqid=0 2024-11-11T07:24:56,647 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/e4fd2d1216424546a0f5fcea6c4fe6e5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/e4fd2d1216424546a0f5fcea6c4fe6e5 2024-11-11T07:24:56,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742419_1595 (size=31651) 2024-11-11T07:24:56,651 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into e4fd2d1216424546a0f5fcea6c4fe6e5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:56,651 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:56,651 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=13, startTime=1731309896200; duration=0sec 2024-11-11T07:24:56,652 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:56,652 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:24:56,652 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:24:56,652 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:24:56,652 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:24:56,653 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:56,653 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1e11cc4b141a43faafd51571a40b24d5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/710ef110e543408e912ea328330923da, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/3dc3eb278db14f3c9ac8dcc900b81aae] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=36.0 K 2024-11-11T07:24:56,653 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e11cc4b141a43faafd51571a40b24d5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731309892998 2024-11-11T07:24:56,653 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/8a1836b4088b4717900a1d193435f344 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8a1836b4088b4717900a1d193435f344 2024-11-11T07:24:56,653 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 710ef110e543408e912ea328330923da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731309893035 2024-11-11T07:24:56,654 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc3eb278db14f3c9ac8dcc900b81aae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731309895204 2024-11-11T07:24:56,658 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into 8a1836b4088b4717900a1d193435f344(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:56,658 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:56,658 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=13, startTime=1731309896199; duration=0sec 2024-11-11T07:24:56,658 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:56,658 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:24:56,660 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:56,661 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/882edd3a9a9b47d29423be0249f3da8d is 50, key is test_row_0/C:col10/1731309895204/Put/seqid=0 2024-11-11T07:24:56,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742420_1596 (size=12697) 2024-11-11T07:24:56,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309956661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309956662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309956871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309956871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:56,947 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:56,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d11177518ca432e8ae12bcddf14dc9d_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d11177518ca432e8ae12bcddf14dc9d_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:56,954 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7373e6095b47492584e1c159af3c9919, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:56,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7373e6095b47492584e1c159af3c9919 is 175, key is test_row_0/A:col10/1731309895368/Put/seqid=0 2024-11-11T07:24:56,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742421_1597 (size=39749) 2024-11-11T07:24:57,068 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/882edd3a9a9b47d29423be0249f3da8d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/882edd3a9a9b47d29423be0249f3da8d 2024-11-11T07:24:57,072 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into 882edd3a9a9b47d29423be0249f3da8d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:57,072 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:57,072 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=13, startTime=1731309896200; duration=0sec 2024-11-11T07:24:57,072 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:57,072 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:24:57,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309957181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309957181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:57,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309957211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:57,215 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309957222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:57,224 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:24:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T07:24:57,251 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:57,252 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-11T07:24:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:24:57,255 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:57,255 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:57,255 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:57,359 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7373e6095b47492584e1c159af3c9919 2024-11-11T07:24:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:24:57,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/7edffb2d73504523b482c04656ede921 is 50, key is test_row_0/B:col10/1731309895368/Put/seqid=0 2024-11-11T07:24:57,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742422_1598 (size=12151) 2024-11-11T07:24:57,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:24:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:57,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:24:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:57,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:24:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309957691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:57,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309957691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:57,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:24:57,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:57,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:57,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:57,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:57,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/7edffb2d73504523b482c04656ede921 2024-11-11T07:24:57,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/e0d50bc704804befac6463d6c5b2cc65 is 50, key is test_row_0/C:col10/1731309895368/Put/seqid=0 2024-11-11T07:24:57,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742423_1599 (size=12151) 2024-11-11T07:24:57,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/e0d50bc704804befac6463d6c5b2cc65 2024-11-11T07:24:57,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7373e6095b47492584e1c159af3c9919 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7373e6095b47492584e1c159af3c9919 2024-11-11T07:24:57,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7373e6095b47492584e1c159af3c9919, entries=200, sequenceid=249, filesize=38.8 K 2024-11-11T07:24:57,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/7edffb2d73504523b482c04656ede921 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/7edffb2d73504523b482c04656ede921 2024-11-11T07:24:57,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/7edffb2d73504523b482c04656ede921, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T07:24:57,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/e0d50bc704804befac6463d6c5b2cc65 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e0d50bc704804befac6463d6c5b2cc65 2024-11-11T07:24:57,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e0d50bc704804befac6463d6c5b2cc65, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T07:24:57,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 022a345c4df8dd612d037e1eac2d4556 in 1304ms, sequenceid=249, compaction requested=false 2024-11-11T07:24:57,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:57,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:24:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:57,867 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:24:57,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c831c71bdb04956bd86043e24cf5812_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309896552/Put/seqid=0 2024-11-11T07:24:57,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742424_1600 (size=12454) 2024-11-11T07:24:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:57,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:24:57,882 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c831c71bdb04956bd86043e24cf5812_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c831c71bdb04956bd86043e24cf5812_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d37eb9843ab43938f61d12a6b20cbe5, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d37eb9843ab43938f61d12a6b20cbe5 is 175, key is test_row_0/A:col10/1731309896552/Put/seqid=0 2024-11-11T07:24:57,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742425_1601 (size=31255) 2024-11-11T07:24:58,288 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d37eb9843ab43938f61d12a6b20cbe5 2024-11-11T07:24:58,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/c5e7e16d456d4f3795defbd714b3d47b is 50, key is test_row_0/B:col10/1731309896552/Put/seqid=0 2024-11-11T07:24:58,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742426_1602 (size=12301) 2024-11-11T07:24:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:24:58,700 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/c5e7e16d456d4f3795defbd714b3d47b 2024-11-11T07:24:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:58,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:58,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/f5e6814fa24e4bc9b1f3573e738c7587 is 50, key is test_row_0/C:col10/1731309896552/Put/seqid=0 2024-11-11T07:24:58,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742427_1603 (size=12301) 2024-11-11T07:24:58,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309958753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:58,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309958757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:58,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309958861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309958871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309959071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309959082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,112 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/f5e6814fa24e4bc9b1f3573e738c7587 2024-11-11T07:24:59,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/7d37eb9843ab43938f61d12a6b20cbe5 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d37eb9843ab43938f61d12a6b20cbe5 2024-11-11T07:24:59,119 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d37eb9843ab43938f61d12a6b20cbe5, entries=150, sequenceid=266, filesize=30.5 K 2024-11-11T07:24:59,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/c5e7e16d456d4f3795defbd714b3d47b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/c5e7e16d456d4f3795defbd714b3d47b 2024-11-11T07:24:59,124 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/c5e7e16d456d4f3795defbd714b3d47b, entries=150, sequenceid=266, filesize=12.0 K 2024-11-11T07:24:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/f5e6814fa24e4bc9b1f3573e738c7587 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f5e6814fa24e4bc9b1f3573e738c7587 2024-11-11T07:24:59,135 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f5e6814fa24e4bc9b1f3573e738c7587, entries=150, sequenceid=266, filesize=12.0 K 2024-11-11T07:24:59,136 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 022a345c4df8dd612d037e1eac2d4556 in 1269ms, sequenceid=266, compaction requested=true 2024-11-11T07:24:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-11T07:24:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-11T07:24:59,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T07:24:59,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8830 sec 2024-11-11T07:24:59,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8890 sec 2024-11-11T07:24:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:59,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:24:59,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:24:59,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111107576397dfc14004aa54b3b2a665c1d1_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309899383/Put/seqid=0 2024-11-11T07:24:59,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742428_1604 (size=14994) 2024-11-11T07:24:59,398 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111107576397dfc14004aa54b3b2a665c1d1_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111107576397dfc14004aa54b3b2a665c1d1_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:24:59,402 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:24:59,402 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:24:59,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/cea771298fd44366b9a2b73f382ace2d, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-11T07:24:59,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/cea771298fd44366b9a2b73f382ace2d is 175, key is test_row_0/A:col10/1731309899383/Put/seqid=0 2024-11-11T07:24:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:24:59,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:24:59,405 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:24:59,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:24:59,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742429_1605 (size=39949) 2024-11-11T07:24:59,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/cea771298fd44366b9a2b73f382ace2d 2024-11-11T07:24:59,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/9949dc3ebf9447ef80163f51b512152d is 50, key is test_row_0/B:col10/1731309899383/Put/seqid=0 2024-11-11T07:24:59,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742430_1606 (size=12301) 2024-11-11T07:24:59,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309959418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309959419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:24:59,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309959531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309959531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:24:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:59,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:59,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:24:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:24:59,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:59,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:24:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:24:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309959741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:24:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309959741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:24:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:24:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:24:59,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/9949dc3ebf9447ef80163f51b512152d 2024-11-11T07:24:59,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/7d7b21e7756d4f4fb265473ce20ac15a is 50, key is test_row_0/C:col10/1731309899383/Put/seqid=0 2024-11-11T07:24:59,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742431_1607 (size=12301) 2024-11-11T07:24:59,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/7d7b21e7756d4f4fb265473ce20ac15a 2024-11-11T07:24:59,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/cea771298fd44366b9a2b73f382ace2d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cea771298fd44366b9a2b73f382ace2d 2024-11-11T07:24:59,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cea771298fd44366b9a2b73f382ace2d, entries=200, sequenceid=289, filesize=39.0 K 2024-11-11T07:24:59,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/9949dc3ebf9447ef80163f51b512152d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/9949dc3ebf9447ef80163f51b512152d 2024-11-11T07:24:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/9949dc3ebf9447ef80163f51b512152d, entries=150, sequenceid=289, filesize=12.0 K 2024-11-11T07:24:59,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/7d7b21e7756d4f4fb265473ce20ac15a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/7d7b21e7756d4f4fb265473ce20ac15a 2024-11-11T07:24:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/7d7b21e7756d4f4fb265473ce20ac15a, entries=150, sequenceid=289, filesize=12.0 K 2024-11-11T07:24:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 022a345c4df8dd612d037e1eac2d4556 in 472ms, sequenceid=289, compaction requested=true 2024-11-11T07:24:59,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:59,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:24:59,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:59,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:24:59,857 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:59,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:59,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:24:59,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:24:59,857 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,858 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:59,858 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:24:59,858 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,858 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/e4fd2d1216424546a0f5fcea6c4fe6e5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/7edffb2d73504523b482c04656ede921, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/c5e7e16d456d4f3795defbd714b3d47b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/9949dc3ebf9447ef80163f51b512152d] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=48.3 K 2024-11-11T07:24:59,859 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:59,859 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:24:59,859 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,859 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8a1836b4088b4717900a1d193435f344, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7373e6095b47492584e1c159af3c9919, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d37eb9843ab43938f61d12a6b20cbe5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cea771298fd44366b9a2b73f382ace2d] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=139.3 K 2024-11-11T07:24:59,859 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,859 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8a1836b4088b4717900a1d193435f344, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7373e6095b47492584e1c159af3c9919, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d37eb9843ab43938f61d12a6b20cbe5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cea771298fd44366b9a2b73f382ace2d] 2024-11-11T07:24:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:24:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,863 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T07:24:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,865 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e4fd2d1216424546a0f5fcea6c4fe6e5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731309895204 2024-11-11T07:24:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,865 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a1836b4088b4717900a1d193435f344, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731309895204 2024-11-11T07:24:59,865 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7edffb2d73504523b482c04656ede921, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731309895368 2024-11-11T07:24:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,865 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7373e6095b47492584e1c159af3c9919, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731309895361 2024-11-11T07:24:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,866 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting c5e7e16d456d4f3795defbd714b3d47b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731309896539 2024-11-11T07:24:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,867 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d37eb9843ab43938f61d12a6b20cbe5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731309896539 2024-11-11T07:24:59,868 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 9949dc3ebf9447ef80163f51b512152d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731309898752 2024-11-11T07:24:59,868 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea771298fd44366b9a2b73f382ace2d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731309898746 2024-11-11T07:24:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113bace91e2e3f418a9195737ea10e565b_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309899405/Put/seqid=0 2024-11-11T07:24:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,897 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:59,897 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6b2e3474f7104134b5ac7340c7bbaca8 is 50, key is test_row_0/B:col10/1731309899383/Put/seqid=0 2024-11-11T07:24:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,905 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,921 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b9943b10080f4b55af51110cc3fc230e_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,924 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b9943b10080f4b55af51110cc3fc230e_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,924 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b9943b10080f4b55af51110cc3fc230e_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742433_1609 (size=12983) 2024-11-11T07:24:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742432_1608 (size=9914) 2024-11-11T07:24:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742434_1610 (size=4469) 2024-11-11T07:24:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,936 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#514 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,936 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/fe4f1cb46a4041e5897dc42e4b00f9cf is 175, key is test_row_0/A:col10/1731309899383/Put/seqid=0 2024-11-11T07:24:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,937 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6b2e3474f7104134b5ac7340c7bbaca8 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6b2e3474f7104134b5ac7340c7bbaca8 2024-11-11T07:24:59,937 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113bace91e2e3f418a9195737ea10e565b_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113bace91e2e3f418a9195737ea10e565b_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:24:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/9b912bf1fd2c48b18f9eba9beacdd027, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:24:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742435_1611 (size=31937) 2024-11-11T07:24:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/9b912bf1fd2c48b18f9eba9beacdd027 is 175, key is test_row_0/A:col10/1731309899405/Put/seqid=0 2024-11-11T07:24:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,944 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into 6b2e3474f7104134b5ac7340c7bbaca8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:59,944 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:59,944 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=12, startTime=1731309899857; duration=0sec 2024-11-11T07:24:59,944 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:24:59,944 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:24:59,944 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:24:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,945 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:24:59,945 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:24:59,946 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:24:59,946 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/882edd3a9a9b47d29423be0249f3da8d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e0d50bc704804befac6463d6c5b2cc65, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f5e6814fa24e4bc9b1f3573e738c7587, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/7d7b21e7756d4f4fb265473ce20ac15a] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=48.3 K 2024-11-11T07:24:59,946 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 882edd3a9a9b47d29423be0249f3da8d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731309895204 2024-11-11T07:24:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742436_1612 (size=22561) 2024-11-11T07:24:59,947 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d50bc704804befac6463d6c5b2cc65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731309895368 2024-11-11T07:24:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,948 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e6814fa24e4bc9b1f3573e738c7587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731309896539 2024-11-11T07:24:59,948 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/9b912bf1fd2c48b18f9eba9beacdd027 2024-11-11T07:24:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,951 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7b21e7756d4f4fb265473ce20ac15a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731309898752 2024-11-11T07:24:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,951 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/fe4f1cb46a4041e5897dc42e4b00f9cf as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/fe4f1cb46a4041e5897dc42e4b00f9cf 2024-11-11T07:24:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/a2cffc464fdb4be9b3e260c05893db38 is 50, key is test_row_0/B:col10/1731309899405/Put/seqid=0 2024-11-11T07:24:59,958 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into fe4f1cb46a4041e5897dc42e4b00f9cf(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:24:59,958 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:24:59,959 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=12, startTime=1731309899857; duration=0sec 2024-11-11T07:24:59,959 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:24:59,959 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:24:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,971 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:24:59,972 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/47d06a4c3024461489ed1d330edbbaba is 50, key is test_row_0/C:col10/1731309899383/Put/seqid=0 2024-11-11T07:24:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742438_1614 (size=12983) 2024-11-11T07:24:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742437_1613 (size=9857) 2024-11-11T07:24:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,998 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/47d06a4c3024461489ed1d330edbbaba as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/47d06a4c3024461489ed1d330edbbaba 2024-11-11T07:24:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:24:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,005 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into 47d06a4c3024461489ed1d330edbbaba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:25:00,005 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:00,005 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=12, startTime=1731309899857; duration=0sec 2024-11-11T07:25:00,005 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:00,005 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:25:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:25:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:00,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309960109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309960111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309960221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309960224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,397 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/a2cffc464fdb4be9b3e260c05893db38 2024-11-11T07:25:00,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/b34da6ff9b714b75bcc0308b8760b75d is 50, key is test_row_0/C:col10/1731309899405/Put/seqid=0 2024-11-11T07:25:00,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742439_1615 (size=9857) 2024-11-11T07:25:00,409 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/b34da6ff9b714b75bcc0308b8760b75d 2024-11-11T07:25:00,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/9b912bf1fd2c48b18f9eba9beacdd027 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/9b912bf1fd2c48b18f9eba9beacdd027 2024-11-11T07:25:00,420 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/9b912bf1fd2c48b18f9eba9beacdd027, entries=100, sequenceid=302, filesize=22.0 K 2024-11-11T07:25:00,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/a2cffc464fdb4be9b3e260c05893db38 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/a2cffc464fdb4be9b3e260c05893db38 2024-11-11T07:25:00,426 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/a2cffc464fdb4be9b3e260c05893db38, entries=100, sequenceid=302, filesize=9.6 K 2024-11-11T07:25:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/b34da6ff9b714b75bcc0308b8760b75d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b34da6ff9b714b75bcc0308b8760b75d 2024-11-11T07:25:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309960432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309960432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,436 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b34da6ff9b714b75bcc0308b8760b75d, entries=100, sequenceid=302, filesize=9.6 K 2024-11-11T07:25:00,438 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 022a345c4df8dd612d037e1eac2d4556 in 575ms, sequenceid=302, compaction requested=false 2024-11-11T07:25:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:00,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-11T07:25:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-11T07:25:00,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-11T07:25:00,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-11T07:25:00,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0440 sec 2024-11-11T07:25:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:25:00,542 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:25:00,543 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:25:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-11T07:25:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:25:00,544 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:25:00,545 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:25:00,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:25:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:25:00,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T07:25:00,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:00,698 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T07:25:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5253a30699447109d77a5bed67c57bb_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309900107/Put/seqid=0 2024-11-11T07:25:00,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742440_1616 (size=12454) 2024-11-11T07:25:00,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:00,714 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5253a30699447109d77a5bed67c57bb_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5253a30699447109d77a5bed67c57bb_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/16f8d2226bd04aedabe08ae7f156ccad, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/16f8d2226bd04aedabe08ae7f156ccad is 175, key is test_row_0/A:col10/1731309900107/Put/seqid=0 2024-11-11T07:25:00,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742441_1617 (size=31255) 2024-11-11T07:25:00,719 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/16f8d2226bd04aedabe08ae7f156ccad 2024-11-11T07:25:00,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/dd4b96c5cea445a9817640e69f69e9ff is 50, key is test_row_0/B:col10/1731309900107/Put/seqid=0 2024-11-11T07:25:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742442_1618 (size=12301) 2024-11-11T07:25:00,731 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/dd4b96c5cea445a9817640e69f69e9ff 2024-11-11T07:25:00,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/e6e3f00ac6ff466f89c4a03e645c6e9b is 50, key is test_row_0/C:col10/1731309900107/Put/seqid=0 2024-11-11T07:25:00,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742443_1619 (size=12301) 2024-11-11T07:25:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:00,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309960760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309960764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:25:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309960872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309960872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309961082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309961082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,143 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/e6e3f00ac6ff466f89c4a03e645c6e9b 2024-11-11T07:25:01,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/16f8d2226bd04aedabe08ae7f156ccad as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/16f8d2226bd04aedabe08ae7f156ccad 2024-11-11T07:25:01,149 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/16f8d2226bd04aedabe08ae7f156ccad, entries=150, sequenceid=328, filesize=30.5 K 2024-11-11T07:25:01,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/dd4b96c5cea445a9817640e69f69e9ff as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dd4b96c5cea445a9817640e69f69e9ff 2024-11-11T07:25:01,156 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dd4b96c5cea445a9817640e69f69e9ff, entries=150, sequenceid=328, filesize=12.0 K 2024-11-11T07:25:01,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/e6e3f00ac6ff466f89c4a03e645c6e9b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e6e3f00ac6ff466f89c4a03e645c6e9b 2024-11-11T07:25:01,160 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e6e3f00ac6ff466f89c4a03e645c6e9b, entries=150, sequenceid=328, filesize=12.0 K 2024-11-11T07:25:01,161 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 022a345c4df8dd612d037e1eac2d4556 in 463ms, sequenceid=328, compaction requested=true 2024-11-11T07:25:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-11T07:25:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-11T07:25:01,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-11T07:25:01,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 617 msec 2024-11-11T07:25:01,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 621 msec 2024-11-11T07:25:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:25:01,172 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:25:01,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:25:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-11T07:25:01,173 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:25:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:25:01,174 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:25:01,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:25:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:01,235 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:25:01,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T07:25:01,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115114d5985a8d468c8f23ed4bcdeef41d_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309900762/Put/seqid=0 2024-11-11T07:25:01,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742444_1620 (size=14994) 2024-11-11T07:25:01,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:01,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115114d5985a8d468c8f23ed4bcdeef41d_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115114d5985a8d468c8f23ed4bcdeef41d_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:01,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be08534fd9e74382885ae06ee6230cde, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:01,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be08534fd9e74382885ae06ee6230cde is 175, key is test_row_0/A:col10/1731309900762/Put/seqid=0 2024-11-11T07:25:01,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742445_1621 (size=39949) 2024-11-11T07:25:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:25:01,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309961290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309961294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:25:01,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309961392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309961392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309961403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309961403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:25:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:25:01,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309961612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309961612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:25:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,657 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be08534fd9e74382885ae06ee6230cde 2024-11-11T07:25:01,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8b1075399d5548a490654d7a8598288b is 50, key is test_row_0/B:col10/1731309900762/Put/seqid=0 2024-11-11T07:25:01,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742446_1622 (size=12301) 2024-11-11T07:25:01,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8b1075399d5548a490654d7a8598288b 2024-11-11T07:25:01,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/d65ce2cc6b924e26a516cf27ca657c53 is 50, key is test_row_0/C:col10/1731309900762/Put/seqid=0 2024-11-11T07:25:01,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742447_1623 (size=12301) 2024-11-11T07:25:01,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:25:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:25:01,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309961901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309961911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309961922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309961922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:01,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:25:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:01,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:02,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/d65ce2cc6b924e26a516cf27ca657c53 2024-11-11T07:25:02,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/be08534fd9e74382885ae06ee6230cde as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be08534fd9e74382885ae06ee6230cde 2024-11-11T07:25:02,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be08534fd9e74382885ae06ee6230cde, entries=200, sequenceid=342, filesize=39.0 K 2024-11-11T07:25:02,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/8b1075399d5548a490654d7a8598288b as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8b1075399d5548a490654d7a8598288b 2024-11-11T07:25:02,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8b1075399d5548a490654d7a8598288b, entries=150, sequenceid=342, filesize=12.0 K 2024-11-11T07:25:02,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/d65ce2cc6b924e26a516cf27ca657c53 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d65ce2cc6b924e26a516cf27ca657c53 2024-11-11T07:25:02,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:25:02,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:02,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:02,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d65ce2cc6b924e26a516cf27ca657c53, entries=150, sequenceid=342, filesize=12.0 K 2024-11-11T07:25:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:02,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 022a345c4df8dd612d037e1eac2d4556 in 856ms, sequenceid=342, compaction requested=true 2024-11-11T07:25:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:25:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:02,092 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:25:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:25:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:02,092 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:25:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:25:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:25:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:25:02,093 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:25:02,093 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:25:02,093 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:02,093 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/fe4f1cb46a4041e5897dc42e4b00f9cf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/9b912bf1fd2c48b18f9eba9beacdd027, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/16f8d2226bd04aedabe08ae7f156ccad, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be08534fd9e74382885ae06ee6230cde] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=122.8 K 2024-11-11T07:25:02,093 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:02,093 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/fe4f1cb46a4041e5897dc42e4b00f9cf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/9b912bf1fd2c48b18f9eba9beacdd027, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/16f8d2226bd04aedabe08ae7f156ccad, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be08534fd9e74382885ae06ee6230cde] 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe4f1cb46a4041e5897dc42e4b00f9cf, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731309898752 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:25:02,095 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:02,095 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6b2e3474f7104134b5ac7340c7bbaca8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/a2cffc464fdb4be9b3e260c05893db38, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dd4b96c5cea445a9817640e69f69e9ff, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8b1075399d5548a490654d7a8598288b] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=46.3 K 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b912bf1fd2c48b18f9eba9beacdd027, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731309899405 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f8d2226bd04aedabe08ae7f156ccad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731309900102 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b2e3474f7104134b5ac7340c7bbaca8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731309898752 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting be08534fd9e74382885ae06ee6230cde, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731309900750 2024-11-11T07:25:02,095 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cffc464fdb4be9b3e260c05893db38, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731309899405 2024-11-11T07:25:02,096 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting dd4b96c5cea445a9817640e69f69e9ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731309900102 2024-11-11T07:25:02,096 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1075399d5548a490654d7a8598288b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731309900753 2024-11-11T07:25:02,116 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:02,118 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:25:02,119 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/4edd4281b2544b94bc67d245c0dfff36 is 50, key is test_row_0/B:col10/1731309900762/Put/seqid=0 2024-11-11T07:25:02,120 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110e533bd9e91942f2b1565e14ccd0b17f_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:02,122 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110e533bd9e91942f2b1565e14ccd0b17f_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:02,122 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110e533bd9e91942f2b1565e14ccd0b17f_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:02,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742449_1625 (size=4469) 2024-11-11T07:25:02,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742448_1624 (size=13119) 2024-11-11T07:25:02,136 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#524 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:25:02,136 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/eb592d0d994b4cf98e303a2b4a12caf3 is 175, key is test_row_0/A:col10/1731309900762/Put/seqid=0 2024-11-11T07:25:02,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742450_1626 (size=32073) 2024-11-11T07:25:02,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39329 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:25:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:02,243 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T07:25:02,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111724b5e7af2a64e0e96a747fdca672071_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309901270/Put/seqid=0 2024-11-11T07:25:02,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742451_1627 (size=12454) 2024-11-11T07:25:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:25:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. as already flushing 2024-11-11T07:25:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309962458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309962461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,541 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/4edd4281b2544b94bc67d245c0dfff36 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/4edd4281b2544b94bc67d245c0dfff36 2024-11-11T07:25:02,543 INFO [Thread-2373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2373. 2024-11-11T07:25:02,543 DEBUG [Thread-2373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:02,543 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,543 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:02,546 INFO [Thread-2375 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2375. 2024-11-11T07:25:02,546 DEBUG [Thread-2375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:02,546 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,547 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,547 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-11-11T07:25:02,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:02,547 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-11-11T07:25:02,547 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:02,547 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,547 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:02,547 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,547 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,547 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:02,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:02,548 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/eb592d0d994b4cf98e303a2b4a12caf3 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/eb592d0d994b4cf98e303a2b4a12caf3 2024-11-11T07:25:02,549 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-11-11T07:25:02,549 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:02,549 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,549 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:02,549 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:02,550 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into 4edd4281b2544b94bc67d245c0dfff36(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:25:02,550 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:02,550 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=12, startTime=1731309902092; duration=0sec 2024-11-11T07:25:02,550 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:25:02,550 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:25:02,550 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:25:02,551 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:25:02,551 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:25:02,551 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:02,551 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/47d06a4c3024461489ed1d330edbbaba, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b34da6ff9b714b75bcc0308b8760b75d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e6e3f00ac6ff466f89c4a03e645c6e9b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d65ce2cc6b924e26a516cf27ca657c53] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=46.3 K 2024-11-11T07:25:02,551 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d06a4c3024461489ed1d330edbbaba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731309898752 2024-11-11T07:25:02,551 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting b34da6ff9b714b75bcc0308b8760b75d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731309899405 2024-11-11T07:25:02,551 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e3f00ac6ff466f89c4a03e645c6e9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731309900102 2024-11-11T07:25:02,552 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting d65ce2cc6b924e26a516cf27ca657c53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731309900753 2024-11-11T07:25:02,552 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into eb592d0d994b4cf98e303a2b4a12caf3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:25:02,552 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:02,552 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=12, startTime=1731309902092; duration=0sec 2024-11-11T07:25:02,552 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:02,552 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:25:02,556 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:25:02,557 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/cd9c87ae4f994a5e8cd820b1324f99db is 50, key is test_row_0/C:col10/1731309900762/Put/seqid=0 2024-11-11T07:25:02,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742452_1628 (size=13119) 2024-11-11T07:25:02,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309962572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309962572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:02,659 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111724b5e7af2a64e0e96a747fdca672071_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111724b5e7af2a64e0e96a747fdca672071_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:02,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/8efcd4c93eae46aa80cdea1b87c096f1, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:02,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/8efcd4c93eae46aa80cdea1b87c096f1 is 175, key is test_row_0/A:col10/1731309901270/Put/seqid=0 2024-11-11T07:25:02,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742453_1629 (size=31255) 2024-11-11T07:25:02,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309962782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309962782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1731309962922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54962 deadline: 1731309962922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:02,923 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_1' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:25:02,923 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_0' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:25:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:02,963 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/cd9c87ae4f994a5e8cd820b1324f99db as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/cd9c87ae4f994a5e8cd820b1324f99db 2024-11-11T07:25:02,967 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into cd9c87ae4f994a5e8cd820b1324f99db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:25:02,967 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:02,967 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=12, startTime=1731309902092; duration=0sec 2024-11-11T07:25:02,967 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:02,967 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:25:03,064 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/8efcd4c93eae46aa80cdea1b87c096f1 2024-11-11T07:25:03,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6c5dae91634a45baa9d4143978fe767a is 50, key is test_row_0/B:col10/1731309901270/Put/seqid=0 2024-11-11T07:25:03,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742454_1630 (size=12301) 2024-11-11T07:25:03,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:03,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309963091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309963091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:25:03,472 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6c5dae91634a45baa9d4143978fe767a 2024-11-11T07:25:03,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/733d6c4b587e4c32813d2153cc7a57fd is 50, key is test_row_0/C:col10/1731309901270/Put/seqid=0 2024-11-11T07:25:03,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742455_1631 (size=12301) 2024-11-11T07:25:03,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55022 deadline: 1731309963481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:03,482 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to f334e8b0fbdf,39329,1731309728191 for 'test_row_2' in 022a345c4df8dd612d037e1eac2d4556 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18312 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T07:25:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T07:25:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54994 deadline: 1731309963601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39329 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1731309963602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 , the old value is region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:25:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., hostname=f334e8b0fbdf,39329,1731309728191, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:25:03,881 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/733d6c4b587e4c32813d2153cc7a57fd 2024-11-11T07:25:03,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/8efcd4c93eae46aa80cdea1b87c096f1 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8efcd4c93eae46aa80cdea1b87c096f1 2024-11-11T07:25:03,888 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8efcd4c93eae46aa80cdea1b87c096f1, entries=150, sequenceid=364, filesize=30.5 K 2024-11-11T07:25:03,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6c5dae91634a45baa9d4143978fe767a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c5dae91634a45baa9d4143978fe767a 2024-11-11T07:25:03,890 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c5dae91634a45baa9d4143978fe767a, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T07:25:03,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/733d6c4b587e4c32813d2153cc7a57fd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/733d6c4b587e4c32813d2153cc7a57fd 2024-11-11T07:25:03,894 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/733d6c4b587e4c32813d2153cc7a57fd, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T07:25:03,895 INFO [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 022a345c4df8dd612d037e1eac2d4556 in 1651ms, sequenceid=364, compaction requested=false 2024-11-11T07:25:03,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:03,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:03,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f334e8b0fbdf:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-11T07:25:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-11T07:25:03,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-11T07:25:03,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7220 sec 2024-11-11T07:25:03,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.7260 sec 2024-11-11T07:25:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39329 {}] regionserver.HRegion(8855): Flush requested on 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:04,613 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T07:25:04,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T07:25:04,613 INFO [Thread-2358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2358. 2024-11-11T07:25:04,613 INFO [Thread-2362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2362. 2024-11-11T07:25:04,613 DEBUG [Thread-2358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:04,613 DEBUG [Thread-2362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:04,613 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,613 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,613 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,614 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:04,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:04,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113c17e4c025db4aa5bbedc54aca420dfb_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309904612/Put/seqid=0 2024-11-11T07:25:04,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742456_1632 (size=12454) 2024-11-11T07:25:04,942 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-11T07:25:04,942 INFO [Thread-2364 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2364. 2024-11-11T07:25:04,942 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:04,942 DEBUG [Thread-2364 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:04,943 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,943 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,943 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,943 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:04,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:04,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:05,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:05,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113c17e4c025db4aa5bbedc54aca420dfb_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113c17e4c025db4aa5bbedc54aca420dfb_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:05,025 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/537ed2f6970b41aca4116034ca882fea, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:05,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/537ed2f6970b41aca4116034ca882fea is 175, key is test_row_0/A:col10/1731309904612/Put/seqid=0 2024-11-11T07:25:05,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742457_1633 (size=31255) 2024-11-11T07:25:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:25:05,342 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:25:05,428 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/537ed2f6970b41aca4116034ca882fea 2024-11-11T07:25:05,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/dcd26ace4ad44b878fe3faec10a0c164 is 50, key is test_row_0/B:col10/1731309904612/Put/seqid=0 2024-11-11T07:25:05,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742458_1634 (size=12301) 2024-11-11T07:25:05,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/dcd26ace4ad44b878fe3faec10a0c164 2024-11-11T07:25:05,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/79668f5d5b8a428cb952c53a7dbdbcb6 is 50, key is test_row_0/C:col10/1731309904612/Put/seqid=0 2024-11-11T07:25:05,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742459_1635 (size=12301) 2024-11-11T07:25:06,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/79668f5d5b8a428cb952c53a7dbdbcb6 2024-11-11T07:25:06,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/537ed2f6970b41aca4116034ca882fea as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/537ed2f6970b41aca4116034ca882fea 2024-11-11T07:25:06,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/537ed2f6970b41aca4116034ca882fea, entries=150, sequenceid=383, filesize=30.5 K 2024-11-11T07:25:06,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/dcd26ace4ad44b878fe3faec10a0c164 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dcd26ace4ad44b878fe3faec10a0c164 2024-11-11T07:25:06,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dcd26ace4ad44b878fe3faec10a0c164, entries=150, sequenceid=383, filesize=12.0 K 2024-11-11T07:25:06,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/79668f5d5b8a428cb952c53a7dbdbcb6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/79668f5d5b8a428cb952c53a7dbdbcb6 2024-11-11T07:25:06,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/79668f5d5b8a428cb952c53a7dbdbcb6, entries=150, sequenceid=383, filesize=12.0 K 2024-11-11T07:25:06,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=13.42 KB/13740 for 022a345c4df8dd612d037e1eac2d4556 in 1644ms, sequenceid=383, compaction requested=true 2024-11-11T07:25:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:25:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:25:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 022a345c4df8dd612d037e1eac2d4556:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:25:06,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:25:06,257 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:25:06,257 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/A is initiating minor compaction (all files) 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/B is initiating minor compaction (all files) 2024-11-11T07:25:06,258 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/A in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:06,258 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/B in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:06,258 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/eb592d0d994b4cf98e303a2b4a12caf3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8efcd4c93eae46aa80cdea1b87c096f1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/537ed2f6970b41aca4116034ca882fea] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=92.4 K 2024-11-11T07:25:06,258 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/4edd4281b2544b94bc67d245c0dfff36, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c5dae91634a45baa9d4143978fe767a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dcd26ace4ad44b878fe3faec10a0c164] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=36.8 K 2024-11-11T07:25:06,258 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. files: [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/eb592d0d994b4cf98e303a2b4a12caf3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8efcd4c93eae46aa80cdea1b87c096f1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/537ed2f6970b41aca4116034ca882fea] 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 4edd4281b2544b94bc67d245c0dfff36, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731309900753 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb592d0d994b4cf98e303a2b4a12caf3, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731309900753 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efcd4c93eae46aa80cdea1b87c096f1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731309901270 2024-11-11T07:25:06,258 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5dae91634a45baa9d4143978fe767a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731309901270 2024-11-11T07:25:06,259 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 537ed2f6970b41aca4116034ca882fea, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731309902456 2024-11-11T07:25:06,259 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting dcd26ace4ad44b878fe3faec10a0c164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731309902456 2024-11-11T07:25:06,264 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:06,265 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#B#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:25:06,265 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/b221112febc040e29068d8d730652b12 is 50, key is test_row_0/B:col10/1731309904612/Put/seqid=0 2024-11-11T07:25:06,265 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411119b4d2ee43e1b4935bd56e0c6f6a7c49e_022a345c4df8dd612d037e1eac2d4556 store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:06,268 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411119b4d2ee43e1b4935bd56e0c6f6a7c49e_022a345c4df8dd612d037e1eac2d4556, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:06,268 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119b4d2ee43e1b4935bd56e0c6f6a7c49e_022a345c4df8dd612d037e1eac2d4556 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:06,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742460_1636 (size=13221) 2024-11-11T07:25:06,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742461_1637 (size=4469) 2024-11-11T07:25:06,529 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:25:06,672 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#A#compaction#534 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:25:06,673 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/cfa544874e744a76b992a83b11105864 is 175, key is test_row_0/A:col10/1731309904612/Put/seqid=0 2024-11-11T07:25:06,673 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/b221112febc040e29068d8d730652b12 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b221112febc040e29068d8d730652b12 2024-11-11T07:25:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742462_1638 (size=32175) 2024-11-11T07:25:06,677 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/B of 022a345c4df8dd612d037e1eac2d4556 into b221112febc040e29068d8d730652b12(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:25:06,677 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:06,677 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/B, priority=13, startTime=1731309906257; duration=0sec 2024-11-11T07:25:06,677 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:25:06,677 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:B 2024-11-11T07:25:06,677 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:25:06,678 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:25:06,678 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1541): 022a345c4df8dd612d037e1eac2d4556/C is initiating minor compaction (all files) 2024-11-11T07:25:06,678 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 022a345c4df8dd612d037e1eac2d4556/C in TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:06,678 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/cd9c87ae4f994a5e8cd820b1324f99db, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/733d6c4b587e4c32813d2153cc7a57fd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/79668f5d5b8a428cb952c53a7dbdbcb6] into tmpdir=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp, totalSize=36.8 K 2024-11-11T07:25:06,678 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9c87ae4f994a5e8cd820b1324f99db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731309900753 2024-11-11T07:25:06,679 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 733d6c4b587e4c32813d2153cc7a57fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731309901270 2024-11-11T07:25:06,679 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] compactions.Compactor(225): Compacting 79668f5d5b8a428cb952c53a7dbdbcb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731309902456 2024-11-11T07:25:06,684 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 022a345c4df8dd612d037e1eac2d4556#C#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:25:06,684 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/05bdd5c78a434e1bac7645c94b410eaa is 50, key is test_row_0/C:col10/1731309904612/Put/seqid=0 2024-11-11T07:25:06,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742463_1639 (size=13221) 2024-11-11T07:25:06,690 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/05bdd5c78a434e1bac7645c94b410eaa as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/05bdd5c78a434e1bac7645c94b410eaa 2024-11-11T07:25:06,693 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/C of 022a345c4df8dd612d037e1eac2d4556 into 05bdd5c78a434e1bac7645c94b410eaa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:25:06,693 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:06,693 INFO [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/C, priority=13, startTime=1731309906257; duration=0sec 2024-11-11T07:25:06,693 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:06,693 DEBUG [RS:0;f334e8b0fbdf:39329-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:C 2024-11-11T07:25:07,080 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/cfa544874e744a76b992a83b11105864 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cfa544874e744a76b992a83b11105864 2024-11-11T07:25:07,083 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 022a345c4df8dd612d037e1eac2d4556/A of 022a345c4df8dd612d037e1eac2d4556 into cfa544874e744a76b992a83b11105864(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:25:07,083 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 022a345c4df8dd612d037e1eac2d4556: 2024-11-11T07:25:07,083 INFO [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., storeName=022a345c4df8dd612d037e1eac2d4556/A, priority=13, startTime=1731309906257; duration=0sec 2024-11-11T07:25:07,083 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:25:07,083 DEBUG [RS:0;f334e8b0fbdf:39329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 022a345c4df8dd612d037e1eac2d4556:A 2024-11-11T07:25:13,543 INFO [Thread-2360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2360. 2024-11-11T07:25:13,543 DEBUG [Thread-2360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:25:13,543 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:13,543 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:13,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3416 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3383 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3325 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3388 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3384 2024-11-11T07:25:13,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:25:13,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:25:13,543 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:25:13,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:13,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:13,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:13,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:25:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:25:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:25:13,547 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309913546"}]},"ts":"1731309913546"} 2024-11-11T07:25:13,548 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:25:13,548 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:25:13,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:25:13,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, UNASSIGN}] 2024-11-11T07:25:13,550 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, UNASSIGN 2024-11-11T07:25:13,551 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=CLOSING, regionLocation=f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:13,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, UNASSIGN because future has completed 2024-11-11T07:25:13,552 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:25:13,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191}] 2024-11-11T07:25:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:25:13,704 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:13,704 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:25:13,704 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 022a345c4df8dd612d037e1eac2d4556, disabling compactions & flushes 2024-11-11T07:25:13,704 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:13,704 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:13,704 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. after waiting 0 ms 2024-11-11T07:25:13,704 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:13,704 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 022a345c4df8dd612d037e1eac2d4556 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-11T07:25:13,709 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111a3955a34ed44fd98343bb32ab301d74_022a345c4df8dd612d037e1eac2d4556 is 50, key is test_row_0/A:col10/1731309904941/Put/seqid=0 2024-11-11T07:25:13,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742464_1640 (size=12454) 2024-11-11T07:25:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:25:14,114 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:25:14,117 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111a3955a34ed44fd98343bb32ab301d74_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a3955a34ed44fd98343bb32ab301d74_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:14,118 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ea12d212187c4d23beb349310e9512fe, store: [table=TestAcidGuarantees family=A region=022a345c4df8dd612d037e1eac2d4556] 2024-11-11T07:25:14,118 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ea12d212187c4d23beb349310e9512fe is 175, key is test_row_0/A:col10/1731309904941/Put/seqid=0 2024-11-11T07:25:14,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742465_1641 (size=31255) 2024-11-11T07:25:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:25:14,522 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ea12d212187c4d23beb349310e9512fe 2024-11-11T07:25:14,526 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6c29d0c1f27b4f899f6ca4ad4939b15a is 50, key is test_row_0/B:col10/1731309904941/Put/seqid=0 2024-11-11T07:25:14,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742466_1642 (size=12301) 2024-11-11T07:25:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:25:14,930 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6c29d0c1f27b4f899f6ca4ad4939b15a 2024-11-11T07:25:14,935 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/dcf8de0808f5465fad1abb691b5642df is 50, key is test_row_0/C:col10/1731309904941/Put/seqid=0 2024-11-11T07:25:14,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742467_1643 (size=12301) 2024-11-11T07:25:15,339 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/dcf8de0808f5465fad1abb691b5642df 2024-11-11T07:25:15,342 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/A/ea12d212187c4d23beb349310e9512fe as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ea12d212187c4d23beb349310e9512fe 2024-11-11T07:25:15,344 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ea12d212187c4d23beb349310e9512fe, entries=150, sequenceid=392, filesize=30.5 K 2024-11-11T07:25:15,344 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/B/6c29d0c1f27b4f899f6ca4ad4939b15a as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c29d0c1f27b4f899f6ca4ad4939b15a 2024-11-11T07:25:15,347 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c29d0c1f27b4f899f6ca4ad4939b15a, entries=150, sequenceid=392, filesize=12.0 K 2024-11-11T07:25:15,347 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/.tmp/C/dcf8de0808f5465fad1abb691b5642df as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/dcf8de0808f5465fad1abb691b5642df 2024-11-11T07:25:15,349 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/dcf8de0808f5465fad1abb691b5642df, entries=150, sequenceid=392, filesize=12.0 K 2024-11-11T07:25:15,350 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 022a345c4df8dd612d037e1eac2d4556 in 1646ms, sequenceid=392, compaction requested=false 2024-11-11T07:25:15,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/965fa37f5ee34b7f84975c649e604fa5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/a88b877fe8cd46928cabb7e8531b43ce, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/23ba3eb2e86a4e6594db01f4e4bbef3d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ec56daa719d34aa0a0c3a15a72a7c4d6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c00bba6dd4c140beb6e32112dce686d8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/4313fb6b52ed47c6bed787d4be9bc881, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c1be6180b4444ed7888c60c20f172d88, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/73ecb46ea25945768dd19c15d3ceda04, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d7651f92d7f48b5933b8040feb196a1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/5ddc578ef87e4ad2b23cdfc81ae854df, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be4135f1eb8842c99ab255e01303f718, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/356526b0075344dda944a978824b21f7, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/1fd28cbdbfb84a7e9aa375f4730d7ce8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/b58ad82973be456a8ac94676bd9c9165, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/260c8d1bd3b04500a4cec219a563fd15, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/6d2f03eb0b9a4d5eb4e1341ed8682865, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8a1836b4088b4717900a1d193435f344, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7373e6095b47492584e1c159af3c9919, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d37eb9843ab43938f61d12a6b20cbe5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cea771298fd44366b9a2b73f382ace2d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/fe4f1cb46a4041e5897dc42e4b00f9cf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/9b912bf1fd2c48b18f9eba9beacdd027, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/16f8d2226bd04aedabe08ae7f156ccad, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be08534fd9e74382885ae06ee6230cde, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/eb592d0d994b4cf98e303a2b4a12caf3, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8efcd4c93eae46aa80cdea1b87c096f1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/537ed2f6970b41aca4116034ca882fea] to archive 2024-11-11T07:25:15,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:25:15,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/965fa37f5ee34b7f84975c649e604fa5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/965fa37f5ee34b7f84975c649e604fa5 2024-11-11T07:25:15,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/a88b877fe8cd46928cabb7e8531b43ce to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/a88b877fe8cd46928cabb7e8531b43ce 2024-11-11T07:25:15,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/23ba3eb2e86a4e6594db01f4e4bbef3d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/23ba3eb2e86a4e6594db01f4e4bbef3d 2024-11-11T07:25:15,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ec56daa719d34aa0a0c3a15a72a7c4d6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ec56daa719d34aa0a0c3a15a72a7c4d6 2024-11-11T07:25:15,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c00bba6dd4c140beb6e32112dce686d8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c00bba6dd4c140beb6e32112dce686d8 2024-11-11T07:25:15,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/4313fb6b52ed47c6bed787d4be9bc881 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/4313fb6b52ed47c6bed787d4be9bc881 2024-11-11T07:25:15,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c1be6180b4444ed7888c60c20f172d88 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/c1be6180b4444ed7888c60c20f172d88 2024-11-11T07:25:15,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/73ecb46ea25945768dd19c15d3ceda04 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/73ecb46ea25945768dd19c15d3ceda04 2024-11-11T07:25:15,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d7651f92d7f48b5933b8040feb196a1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d7651f92d7f48b5933b8040feb196a1 2024-11-11T07:25:15,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/5ddc578ef87e4ad2b23cdfc81ae854df to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/5ddc578ef87e4ad2b23cdfc81ae854df 2024-11-11T07:25:15,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be4135f1eb8842c99ab255e01303f718 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be4135f1eb8842c99ab255e01303f718 2024-11-11T07:25:15,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/356526b0075344dda944a978824b21f7 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/356526b0075344dda944a978824b21f7 2024-11-11T07:25:15,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/1fd28cbdbfb84a7e9aa375f4730d7ce8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/1fd28cbdbfb84a7e9aa375f4730d7ce8 2024-11-11T07:25:15,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/b58ad82973be456a8ac94676bd9c9165 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/b58ad82973be456a8ac94676bd9c9165 2024-11-11T07:25:15,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/260c8d1bd3b04500a4cec219a563fd15 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/260c8d1bd3b04500a4cec219a563fd15 2024-11-11T07:25:15,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/6d2f03eb0b9a4d5eb4e1341ed8682865 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/6d2f03eb0b9a4d5eb4e1341ed8682865 2024-11-11T07:25:15,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8a1836b4088b4717900a1d193435f344 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8a1836b4088b4717900a1d193435f344 2024-11-11T07:25:15,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7373e6095b47492584e1c159af3c9919 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7373e6095b47492584e1c159af3c9919 2024-11-11T07:25:15,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d37eb9843ab43938f61d12a6b20cbe5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/7d37eb9843ab43938f61d12a6b20cbe5 2024-11-11T07:25:15,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cea771298fd44366b9a2b73f382ace2d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cea771298fd44366b9a2b73f382ace2d 2024-11-11T07:25:15,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/fe4f1cb46a4041e5897dc42e4b00f9cf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/fe4f1cb46a4041e5897dc42e4b00f9cf 2024-11-11T07:25:15,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/9b912bf1fd2c48b18f9eba9beacdd027 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/9b912bf1fd2c48b18f9eba9beacdd027 2024-11-11T07:25:15,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/16f8d2226bd04aedabe08ae7f156ccad to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/16f8d2226bd04aedabe08ae7f156ccad 2024-11-11T07:25:15,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be08534fd9e74382885ae06ee6230cde to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/be08534fd9e74382885ae06ee6230cde 2024-11-11T07:25:15,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/eb592d0d994b4cf98e303a2b4a12caf3 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/eb592d0d994b4cf98e303a2b4a12caf3 2024-11-11T07:25:15,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8efcd4c93eae46aa80cdea1b87c096f1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/8efcd4c93eae46aa80cdea1b87c096f1 2024-11-11T07:25:15,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/537ed2f6970b41aca4116034ca882fea to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/537ed2f6970b41aca4116034ca882fea 2024-11-11T07:25:15,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/ae34953d51ea4f0a96a1acc6ed07d10f, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/668e29b1c2c1459aad6a085d185cc3cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/795127a2e0194500a751433aa3e3fcba, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3d58b4950f2d4e86a37f9b307a701cd4, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3dfceb1954b44bd7b74ca96d684d1b0e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/f1f91ce995064c59a4d55510d01aead1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/673a4bd69a6d4cfeae5b98cfc5fb99f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8fd57213cfbc4b2081143246b8ec710b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b01c10cb59ea42d0b2c26438cb3bc5c1, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/07095ec554b6416890759945d782d354, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/2eec20061514410290c0cb4dff2b4e23, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/823f4c86c11f4ab6b2d9e314fd13ca67, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/5293bede63664dcf82eee611a6c6d0cd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/abc1b6750d514a61bee302867ccc7d72, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/e4fd2d1216424546a0f5fcea6c4fe6e5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/bb880f22bee94c58a2ff5953cc3c46df, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/7edffb2d73504523b482c04656ede921, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/c5e7e16d456d4f3795defbd714b3d47b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6b2e3474f7104134b5ac7340c7bbaca8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/9949dc3ebf9447ef80163f51b512152d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/a2cffc464fdb4be9b3e260c05893db38, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dd4b96c5cea445a9817640e69f69e9ff, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/4edd4281b2544b94bc67d245c0dfff36, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8b1075399d5548a490654d7a8598288b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c5dae91634a45baa9d4143978fe767a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dcd26ace4ad44b878fe3faec10a0c164] to archive 2024-11-11T07:25:15,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:25:15,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/ae34953d51ea4f0a96a1acc6ed07d10f to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/ae34953d51ea4f0a96a1acc6ed07d10f 2024-11-11T07:25:15,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/668e29b1c2c1459aad6a085d185cc3cd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/668e29b1c2c1459aad6a085d185cc3cd 2024-11-11T07:25:15,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/795127a2e0194500a751433aa3e3fcba to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/795127a2e0194500a751433aa3e3fcba 2024-11-11T07:25:15,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3d58b4950f2d4e86a37f9b307a701cd4 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3d58b4950f2d4e86a37f9b307a701cd4 2024-11-11T07:25:15,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3dfceb1954b44bd7b74ca96d684d1b0e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/3dfceb1954b44bd7b74ca96d684d1b0e 2024-11-11T07:25:15,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/f1f91ce995064c59a4d55510d01aead1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/f1f91ce995064c59a4d55510d01aead1 2024-11-11T07:25:15,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/673a4bd69a6d4cfeae5b98cfc5fb99f8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/673a4bd69a6d4cfeae5b98cfc5fb99f8 2024-11-11T07:25:15,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8fd57213cfbc4b2081143246b8ec710b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8fd57213cfbc4b2081143246b8ec710b 2024-11-11T07:25:15,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b01c10cb59ea42d0b2c26438cb3bc5c1 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b01c10cb59ea42d0b2c26438cb3bc5c1 2024-11-11T07:25:15,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/07095ec554b6416890759945d782d354 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/07095ec554b6416890759945d782d354 2024-11-11T07:25:15,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/2eec20061514410290c0cb4dff2b4e23 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/2eec20061514410290c0cb4dff2b4e23 2024-11-11T07:25:15,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/823f4c86c11f4ab6b2d9e314fd13ca67 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/823f4c86c11f4ab6b2d9e314fd13ca67 2024-11-11T07:25:15,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/5293bede63664dcf82eee611a6c6d0cd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/5293bede63664dcf82eee611a6c6d0cd 2024-11-11T07:25:15,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8e8f73d0fd4c4b29aa23e7f0678d4f3e 2024-11-11T07:25:15,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/abc1b6750d514a61bee302867ccc7d72 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/abc1b6750d514a61bee302867ccc7d72 2024-11-11T07:25:15,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/e4fd2d1216424546a0f5fcea6c4fe6e5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/e4fd2d1216424546a0f5fcea6c4fe6e5 2024-11-11T07:25:15,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/bb880f22bee94c58a2ff5953cc3c46df to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/bb880f22bee94c58a2ff5953cc3c46df 2024-11-11T07:25:15,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/7edffb2d73504523b482c04656ede921 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/7edffb2d73504523b482c04656ede921 2024-11-11T07:25:15,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/c5e7e16d456d4f3795defbd714b3d47b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/c5e7e16d456d4f3795defbd714b3d47b 2024-11-11T07:25:15,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6b2e3474f7104134b5ac7340c7bbaca8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6b2e3474f7104134b5ac7340c7bbaca8 2024-11-11T07:25:15,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/9949dc3ebf9447ef80163f51b512152d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/9949dc3ebf9447ef80163f51b512152d 2024-11-11T07:25:15,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/a2cffc464fdb4be9b3e260c05893db38 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/a2cffc464fdb4be9b3e260c05893db38 2024-11-11T07:25:15,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dd4b96c5cea445a9817640e69f69e9ff to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dd4b96c5cea445a9817640e69f69e9ff 2024-11-11T07:25:15,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/4edd4281b2544b94bc67d245c0dfff36 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/4edd4281b2544b94bc67d245c0dfff36 2024-11-11T07:25:15,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8b1075399d5548a490654d7a8598288b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/8b1075399d5548a490654d7a8598288b 2024-11-11T07:25:15,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c5dae91634a45baa9d4143978fe767a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c5dae91634a45baa9d4143978fe767a 2024-11-11T07:25:15,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dcd26ace4ad44b878fe3faec10a0c164 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/dcd26ace4ad44b878fe3faec10a0c164 2024-11-11T07:25:15,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/0bab1344c9ad462e8a018ef4439eb206, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/5269da2843bf473db329b516ad8581fd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f8e4593d1ee6497d9a382bfa9af810f8, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/09730a16a3ff43cdb4f1de2ea3728183, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/9d87a3446a8e402ea5ba8437c299310e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1870f83a0e714786b3271640f9c3548e, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b4440c86d3db404aa99230a39e8707d6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/a2c7b63503e640ec8d6dd2a6f7e83df5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/56497654b389405c8ddbb2f18deeb8fe, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d6216f8810704c96a53aa6ad068eabcf, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/de701c7ede1a43c3b91f4e7fe8fcb451, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/bc186568e19e448eb4d17ad3276346ff, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1e11cc4b141a43faafd51571a40b24d5, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/c6c1eef571a445d8aec1a6b0abeeb6e6, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/710ef110e543408e912ea328330923da, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/882edd3a9a9b47d29423be0249f3da8d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/3dc3eb278db14f3c9ac8dcc900b81aae, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e0d50bc704804befac6463d6c5b2cc65, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f5e6814fa24e4bc9b1f3573e738c7587, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/47d06a4c3024461489ed1d330edbbaba, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/7d7b21e7756d4f4fb265473ce20ac15a, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b34da6ff9b714b75bcc0308b8760b75d, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e6e3f00ac6ff466f89c4a03e645c6e9b, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/cd9c87ae4f994a5e8cd820b1324f99db, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d65ce2cc6b924e26a516cf27ca657c53, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/733d6c4b587e4c32813d2153cc7a57fd, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/79668f5d5b8a428cb952c53a7dbdbcb6] to archive 2024-11-11T07:25:15,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:25:15,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/0bab1344c9ad462e8a018ef4439eb206 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/0bab1344c9ad462e8a018ef4439eb206 2024-11-11T07:25:15,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/5269da2843bf473db329b516ad8581fd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/5269da2843bf473db329b516ad8581fd 2024-11-11T07:25:15,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f8e4593d1ee6497d9a382bfa9af810f8 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f8e4593d1ee6497d9a382bfa9af810f8 2024-11-11T07:25:15,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/09730a16a3ff43cdb4f1de2ea3728183 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/09730a16a3ff43cdb4f1de2ea3728183 2024-11-11T07:25:15,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/9d87a3446a8e402ea5ba8437c299310e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/9d87a3446a8e402ea5ba8437c299310e 2024-11-11T07:25:15,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1870f83a0e714786b3271640f9c3548e to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1870f83a0e714786b3271640f9c3548e 2024-11-11T07:25:15,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b4440c86d3db404aa99230a39e8707d6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b4440c86d3db404aa99230a39e8707d6 2024-11-11T07:25:15,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/a2c7b63503e640ec8d6dd2a6f7e83df5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/a2c7b63503e640ec8d6dd2a6f7e83df5 2024-11-11T07:25:15,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/56497654b389405c8ddbb2f18deeb8fe to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/56497654b389405c8ddbb2f18deeb8fe 2024-11-11T07:25:15,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d6216f8810704c96a53aa6ad068eabcf to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d6216f8810704c96a53aa6ad068eabcf 2024-11-11T07:25:15,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/de701c7ede1a43c3b91f4e7fe8fcb451 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/de701c7ede1a43c3b91f4e7fe8fcb451 2024-11-11T07:25:15,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/bc186568e19e448eb4d17ad3276346ff to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/bc186568e19e448eb4d17ad3276346ff 2024-11-11T07:25:15,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1e11cc4b141a43faafd51571a40b24d5 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/1e11cc4b141a43faafd51571a40b24d5 2024-11-11T07:25:15,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/c6c1eef571a445d8aec1a6b0abeeb6e6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/c6c1eef571a445d8aec1a6b0abeeb6e6 2024-11-11T07:25:15,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/710ef110e543408e912ea328330923da to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/710ef110e543408e912ea328330923da 2024-11-11T07:25:15,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/882edd3a9a9b47d29423be0249f3da8d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/882edd3a9a9b47d29423be0249f3da8d 2024-11-11T07:25:15,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/3dc3eb278db14f3c9ac8dcc900b81aae to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/3dc3eb278db14f3c9ac8dcc900b81aae 2024-11-11T07:25:15,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e0d50bc704804befac6463d6c5b2cc65 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e0d50bc704804befac6463d6c5b2cc65 2024-11-11T07:25:15,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f5e6814fa24e4bc9b1f3573e738c7587 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/f5e6814fa24e4bc9b1f3573e738c7587 2024-11-11T07:25:15,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/47d06a4c3024461489ed1d330edbbaba to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/47d06a4c3024461489ed1d330edbbaba 2024-11-11T07:25:15,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/7d7b21e7756d4f4fb265473ce20ac15a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/7d7b21e7756d4f4fb265473ce20ac15a 2024-11-11T07:25:15,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b34da6ff9b714b75bcc0308b8760b75d to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/b34da6ff9b714b75bcc0308b8760b75d 2024-11-11T07:25:15,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e6e3f00ac6ff466f89c4a03e645c6e9b to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/e6e3f00ac6ff466f89c4a03e645c6e9b 2024-11-11T07:25:15,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/cd9c87ae4f994a5e8cd820b1324f99db to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/cd9c87ae4f994a5e8cd820b1324f99db 2024-11-11T07:25:15,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d65ce2cc6b924e26a516cf27ca657c53 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/d65ce2cc6b924e26a516cf27ca657c53 2024-11-11T07:25:15,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/733d6c4b587e4c32813d2153cc7a57fd to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/733d6c4b587e4c32813d2153cc7a57fd 2024-11-11T07:25:15,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/79668f5d5b8a428cb952c53a7dbdbcb6 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/79668f5d5b8a428cb952c53a7dbdbcb6 2024-11-11T07:25:15,421 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/recovered.edits/395.seqid, newMaxSeqId=395, maxSeqId=4 2024-11-11T07:25:15,421 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. 2024-11-11T07:25:15,422 DEBUG [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 022a345c4df8dd612d037e1eac2d4556: Waiting for close lock at 1731309913704Running coprocessor pre-close hooks at 1731309913704Disabling compacts and flushes for region at 1731309913704Disabling writes for close at 1731309913704Obtaining lock to block concurrent updates at 1731309913704Preparing flush snapshotting stores in 022a345c4df8dd612d037e1eac2d4556 at 1731309913704Finished memstore snapshotting TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731309913704Flushing stores of TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556. at 1731309913705 (+1 ms)Flushing 022a345c4df8dd612d037e1eac2d4556/A: creating writer at 1731309913705Flushing mob file 022a345c4df8dd612d037e1eac2d4556/A: appending metadata at 1731309913709 (+4 ms)Flushing mob file 022a345c4df8dd612d037e1eac2d4556/A: closing flushed file at 1731309913709Flushing 022a345c4df8dd612d037e1eac2d4556/A: appending metadata at 1731309914118 (+409 ms)Flushing 022a345c4df8dd612d037e1eac2d4556/A: closing flushed file at 1731309914118Flushing 022a345c4df8dd612d037e1eac2d4556/B: creating writer at 1731309914524 (+406 ms)Flushing 022a345c4df8dd612d037e1eac2d4556/B: appending metadata at 1731309914526 (+2 ms)Flushing 022a345c4df8dd612d037e1eac2d4556/B: closing flushed file at 1731309914526Flushing 022a345c4df8dd612d037e1eac2d4556/C: creating writer at 1731309914933 (+407 ms)Flushing 022a345c4df8dd612d037e1eac2d4556/C: appending metadata at 1731309914935 (+2 ms)Flushing 022a345c4df8dd612d037e1eac2d4556/C: closing flushed file at 1731309914935Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41f74873: reopening flushed file at 1731309915341 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2716b775: reopening flushed file at 1731309915344 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a145875: reopening flushed file at 1731309915347 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 022a345c4df8dd612d037e1eac2d4556 in 1646ms, sequenceid=392, compaction requested=false at 1731309915350 (+3 ms)Writing region close event to WAL at 1731309915418 (+68 ms)Running coprocessor post-close hooks at 1731309915421 (+3 ms)Closed at 1731309915421 2024-11-11T07:25:15,423 INFO [RS_CLOSE_REGION-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=022a345c4df8dd612d037e1eac2d4556, regionState=CLOSED 2024-11-11T07:25:15,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 because future has completed 2024-11-11T07:25:15,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-11T07:25:15,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 022a345c4df8dd612d037e1eac2d4556, server=f334e8b0fbdf,39329,1731309728191 in 1.8730 sec 2024-11-11T07:25:15,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-11T07:25:15,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=022a345c4df8dd612d037e1eac2d4556, UNASSIGN in 1.8760 sec 2024-11-11T07:25:15,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-11T07:25:15,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8800 sec 2024-11-11T07:25:15,430 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731309915430"}]},"ts":"1731309915430"} 2024-11-11T07:25:15,431 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:25:15,431 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:25:15,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8880 sec 2024-11-11T07:25:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:25:15,692 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:25:15,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:25:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:25:15,694 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:25:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:25:15,694 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:25:15,696 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,697 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C, FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/recovered.edits] 2024-11-11T07:25:15,699 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cfa544874e744a76b992a83b11105864 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/cfa544874e744a76b992a83b11105864 2024-11-11T07:25:15,700 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ea12d212187c4d23beb349310e9512fe to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/A/ea12d212187c4d23beb349310e9512fe 2024-11-11T07:25:15,701 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c29d0c1f27b4f899f6ca4ad4939b15a to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/6c29d0c1f27b4f899f6ca4ad4939b15a 2024-11-11T07:25:15,702 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b221112febc040e29068d8d730652b12 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/B/b221112febc040e29068d8d730652b12 2024-11-11T07:25:15,703 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/05bdd5c78a434e1bac7645c94b410eaa to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/05bdd5c78a434e1bac7645c94b410eaa 2024-11-11T07:25:15,704 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/dcf8de0808f5465fad1abb691b5642df to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/C/dcf8de0808f5465fad1abb691b5642df 2024-11-11T07:25:15,705 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/recovered.edits/395.seqid to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556/recovered.edits/395.seqid 2024-11-11T07:25:15,706 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/default/TestAcidGuarantees/022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,706 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:25:15,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:25:15,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T07:25:15,708 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111107576397dfc14004aa54b3b2a665c1d1_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111107576397dfc14004aa54b3b2a665c1d1_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,709 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a3955a34ed44fd98343bb32ab301d74_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a3955a34ed44fd98343bb32ab301d74_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e5457c1ef4b478ab12dd284b8175611_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e5457c1ef4b478ab12dd284b8175611_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112591794f90e24766b8568f17718091e8_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112591794f90e24766b8568f17718091e8_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,711 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113bace91e2e3f418a9195737ea10e565b_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113bace91e2e3f418a9195737ea10e565b_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,712 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113c17e4c025db4aa5bbedc54aca420dfb_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113c17e4c025db4aa5bbedc54aca420dfb_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,713 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b876ff274f24c6f946c3336c28e4c95_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b876ff274f24c6f946c3336c28e4c95_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,713 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b94c7e4d72e49fb97a5bedb7bf9be17_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b94c7e4d72e49fb97a5bedb7bf9be17_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,714 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c831c71bdb04956bd86043e24cf5812_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c831c71bdb04956bd86043e24cf5812_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,715 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115114d5985a8d468c8f23ed4bcdeef41d_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115114d5985a8d468c8f23ed4bcdeef41d_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,715 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163435aea945f47c2ac5967cb8815c3eb_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163435aea945f47c2ac5967cb8815c3eb_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,716 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d11177518ca432e8ae12bcddf14dc9d_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d11177518ca432e8ae12bcddf14dc9d_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,717 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111724b5e7af2a64e0e96a747fdca672071_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111724b5e7af2a64e0e96a747fdca672071_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,717 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118cd0ab4f1fef4d26bab4caa0ccdedd42_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118cd0ab4f1fef4d26bab4caa0ccdedd42_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,718 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a5c80f6ee8b1490789f4e50fb4cadc57_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a5c80f6ee8b1490789f4e50fb4cadc57_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,719 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b360321e3cef4f78aca3ada3c532ee70_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b360321e3cef4f78aca3ada3c532ee70_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,720 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5253a30699447109d77a5bed67c57bb_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5253a30699447109d77a5bed67c57bb_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,721 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d83131656ae94377acb14b0ba860532b_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d83131656ae94377acb14b0ba860532b_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,722 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc1eb48deb514be1b1f2ca57c0ace1cd_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc1eb48deb514be1b1f2ca57c0ace1cd_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,722 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4c1d8a7a8f3490bb65015f72a358b4e_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4c1d8a7a8f3490bb65015f72a358b4e_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,723 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fdce3b081d2d44daabba9e97f8f24fc1_022a345c4df8dd612d037e1eac2d4556 to hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fdce3b081d2d44daabba9e97f8f24fc1_022a345c4df8dd612d037e1eac2d4556 2024-11-11T07:25:15,723 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:25:15,725 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:25:15,727 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:25:15,729 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:25:15,729 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:25:15,730 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:25:15,730 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731309915730"}]},"ts":"9223372036854775807"} 2024-11-11T07:25:15,731 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:25:15,731 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 022a345c4df8dd612d037e1eac2d4556, NAME => 'TestAcidGuarantees,,1731309879423.022a345c4df8dd612d037e1eac2d4556.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:25:15,731 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:25:15,731 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731309915731"}]},"ts":"9223372036854775807"} 2024-11-11T07:25:15,733 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:25:15,733 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:25:15,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-11T07:25:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:25:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:25:15,802 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:25:15,811 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1197 (was 1950), ProcessCount=11 (was 11), AvailableMemoryMB=8132 (was 8956) 2024-11-11T07:25:15,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T07:25:15,811 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:25:15,811 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:25:15,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:15,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:15,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:25:15,811 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T07:25:15,811 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2038551008, stopped=false 2024-11-11T07:25:15,812 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f334e8b0fbdf,39865,1731309727465 2024-11-11T07:25:15,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T07:25:15,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T07:25:15,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:25:15,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:25:15,814 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T07:25:15,814 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:25:15,814 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:25:15,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:15,815 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f334e8b0fbdf,39329,1731309728191' ***** 2024-11-11T07:25:15,815 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:25:15,815 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:25:15,815 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T07:25:15,815 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T07:25:15,816 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T07:25:15,816 INFO [RS:0;f334e8b0fbdf:39329 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T07:25:15,816 INFO [RS:0;f334e8b0fbdf:39329 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T07:25:15,816 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(959): stopping server f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:15,816 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T07:25:15,816 INFO [RS:0;f334e8b0fbdf:39329 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f334e8b0fbdf:39329. 2024-11-11T07:25:15,816 DEBUG [RS:0;f334e8b0fbdf:39329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:25:15,816 DEBUG [RS:0;f334e8b0fbdf:39329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:15,817 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T07:25:15,817 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T07:25:15,817 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T07:25:15,817 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T07:25:15,817 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T07:25:15,817 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T07:25:15,818 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:15,818 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T07:25:15,818 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T07:25:15,818 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T07:25:15,818 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T07:25:15,818 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T07:25:15,818 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T07:25:15,827 INFO [regionserver/f334e8b0fbdf:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T07:25:15,839 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/info/d0a1ed1df9f2474493e88047eb2e355d is 95, key is TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df./info:/1731309758450/DeleteFamily/seqid=0 2024-11-11T07:25:15,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742468_1644 (size=6514) 2024-11-11T07:25:16,018 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:16,034 INFO [regionserver/f334e8b0fbdf:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T07:25:16,035 INFO [regionserver/f334e8b0fbdf:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T07:25:16,218 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:16,243 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/info/d0a1ed1df9f2474493e88047eb2e355d 2024-11-11T07:25:16,261 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/ns/5bfb4599c3ee46559070a9599d92d437 is 93, key is TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df./ns:/1731309758409/DeleteFamily/seqid=0 2024-11-11T07:25:16,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742469_1645 (size=5906) 2024-11-11T07:25:16,418 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:16,618 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:16,664 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/ns/5bfb4599c3ee46559070a9599d92d437 2024-11-11T07:25:16,682 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/rep_barrier/6379362589b246d496201245d96e43b7 is 102, key is TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df./rep_barrier:/1731309758409/DeleteFamily/seqid=0 2024-11-11T07:25:16,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742470_1646 (size=6025) 2024-11-11T07:25:16,819 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:17,019 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:17,085 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/rep_barrier/6379362589b246d496201245d96e43b7 2024-11-11T07:25:17,103 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/table/86d508e7c9534484aa2cb8626dea65cd is 96, key is TestAcidGuarantees,,1731309732506.5a2be9895ea70c6a849961e6f35c48df./table:/1731309758409/DeleteFamily/seqid=0 2024-11-11T07:25:17,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742471_1647 (size=5936) 2024-11-11T07:25:17,219 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:17,419 DEBUG [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:25:17,507 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/table/86d508e7c9534484aa2cb8626dea65cd 2024-11-11T07:25:17,510 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/info/d0a1ed1df9f2474493e88047eb2e355d as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/info/d0a1ed1df9f2474493e88047eb2e355d 2024-11-11T07:25:17,512 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/info/d0a1ed1df9f2474493e88047eb2e355d, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T07:25:17,513 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/ns/5bfb4599c3ee46559070a9599d92d437 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/ns/5bfb4599c3ee46559070a9599d92d437 2024-11-11T07:25:17,515 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/ns/5bfb4599c3ee46559070a9599d92d437, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T07:25:17,515 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/rep_barrier/6379362589b246d496201245d96e43b7 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/rep_barrier/6379362589b246d496201245d96e43b7 2024-11-11T07:25:17,518 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/rep_barrier/6379362589b246d496201245d96e43b7, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T07:25:17,518 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/.tmp/table/86d508e7c9534484aa2cb8626dea65cd as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/table/86d508e7c9534484aa2cb8626dea65cd 2024-11-11T07:25:17,520 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/table/86d508e7c9534484aa2cb8626dea65cd, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T07:25:17,521 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false 2024-11-11T07:25:17,524 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T07:25:17,525 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T07:25:17,525 INFO [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T07:25:17,525 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731309915817Running coprocessor pre-close hooks at 1731309915817Disabling compacts and flushes for region at 1731309915818 (+1 ms)Disabling writes for close at 1731309915818Obtaining lock to block concurrent updates at 1731309915818Preparing flush snapshotting stores in 1588230740 at 1731309915818Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731309915818Flushing stores of hbase:meta,,1.1588230740 at 1731309915818Flushing 1588230740/info: creating writer at 1731309915819 (+1 ms)Flushing 1588230740/info: appending metadata at 1731309915839 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731309915839Flushing 1588230740/ns: creating writer at 1731309916246 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731309916261 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731309916261Flushing 1588230740/rep_barrier: creating writer at 1731309916667 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731309916681 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731309916681Flushing 1588230740/table: creating writer at 1731309917088 (+407 ms)Flushing 1588230740/table: appending metadata at 1731309917103 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731309917103Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2864d66a: reopening flushed file at 1731309917509 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1239fb39: reopening flushed file at 1731309917512 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f8ae6da: reopening flushed file at 1731309917515 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@381f0dd2: reopening flushed file at 1731309917518 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false at 1731309917521 (+3 ms)Writing region close event to WAL at 1731309917522 (+1 ms)Running coprocessor post-close hooks at 1731309917525 (+3 ms)Closed at 1731309917525 2024-11-11T07:25:17,525 DEBUG [RS_CLOSE_META-regionserver/f334e8b0fbdf:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T07:25:17,619 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(976): stopping server f334e8b0fbdf,39329,1731309728191; all regions closed. 2024-11-11T07:25:17,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741834_1010 (size=25344) 2024-11-11T07:25:17,625 DEBUG [RS:0;f334e8b0fbdf:39329 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/oldWALs 2024-11-11T07:25:17,626 INFO [RS:0;f334e8b0fbdf:39329 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f334e8b0fbdf%2C39329%2C1731309728191.meta:.meta(num 1731309732117) 2024-11-11T07:25:17,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741832_1008 (size=14535385) 2024-11-11T07:25:17,629 DEBUG [RS:0;f334e8b0fbdf:39329 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/oldWALs 2024-11-11T07:25:17,629 INFO [RS:0;f334e8b0fbdf:39329 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f334e8b0fbdf%2C39329%2C1731309728191:(num 1731309731141) 2024-11-11T07:25:17,629 DEBUG [RS:0;f334e8b0fbdf:39329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:25:17,629 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T07:25:17,629 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T07:25:17,629 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.ChoreService(370): Chore service for: regionserver/f334e8b0fbdf:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T07:25:17,629 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T07:25:17,629 INFO [regionserver/f334e8b0fbdf:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T07:25:17,630 INFO [RS:0;f334e8b0fbdf:39329 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39329 2024-11-11T07:25:17,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f334e8b0fbdf,39329,1731309728191 2024-11-11T07:25:17,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T07:25:17,634 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T07:25:17,636 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f334e8b0fbdf,39329,1731309728191] 2024-11-11T07:25:17,638 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f334e8b0fbdf,39329,1731309728191 already deleted, retry=false 2024-11-11T07:25:17,638 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f334e8b0fbdf,39329,1731309728191 expired; onlineServers=0 2024-11-11T07:25:17,638 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f334e8b0fbdf,39865,1731309727465' ***** 2024-11-11T07:25:17,638 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T07:25:17,638 INFO [M:0;f334e8b0fbdf:39865 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T07:25:17,638 INFO [M:0;f334e8b0fbdf:39865 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T07:25:17,638 DEBUG [M:0;f334e8b0fbdf:39865 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T07:25:17,638 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T07:25:17,639 DEBUG [M:0;f334e8b0fbdf:39865 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T07:25:17,639 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster-HFileCleaner.large.0-1731309730844 {}] cleaner.HFileCleaner(306): Exit Thread[master/f334e8b0fbdf:0:becomeActiveMaster-HFileCleaner.large.0-1731309730844,5,FailOnTimeoutGroup] 2024-11-11T07:25:17,639 DEBUG [master/f334e8b0fbdf:0:becomeActiveMaster-HFileCleaner.small.0-1731309730857 {}] cleaner.HFileCleaner(306): Exit Thread[master/f334e8b0fbdf:0:becomeActiveMaster-HFileCleaner.small.0-1731309730857,5,FailOnTimeoutGroup] 2024-11-11T07:25:17,639 INFO [M:0;f334e8b0fbdf:39865 {}] hbase.ChoreService(370): Chore service for: master/f334e8b0fbdf:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T07:25:17,639 INFO [M:0;f334e8b0fbdf:39865 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T07:25:17,639 DEBUG [M:0;f334e8b0fbdf:39865 {}] master.HMaster(1795): Stopping service threads 2024-11-11T07:25:17,639 INFO [M:0;f334e8b0fbdf:39865 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T07:25:17,639 INFO [M:0;f334e8b0fbdf:39865 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T07:25:17,639 ERROR [M:0;f334e8b0fbdf:39865 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T07:25:17,640 INFO [M:0;f334e8b0fbdf:39865 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T07:25:17,640 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T07:25:17,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T07:25:17,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:25:17,641 DEBUG [M:0;f334e8b0fbdf:39865 {}] zookeeper.ZKUtil(347): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T07:25:17,641 WARN [M:0;f334e8b0fbdf:39865 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T07:25:17,641 INFO [M:0;f334e8b0fbdf:39865 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/.lastflushedseqids 2024-11-11T07:25:17,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742472_1648 (size=72) 2024-11-11T07:25:17,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:25:17,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39329-0x10105e7d61c0001, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:25:17,737 INFO [RS:0;f334e8b0fbdf:39329 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T07:25:17,737 INFO [RS:0;f334e8b0fbdf:39329 {}] regionserver.HRegionServer(1031): Exiting; stopping=f334e8b0fbdf,39329,1731309728191; zookeeper connection closed. 2024-11-11T07:25:17,738 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6fa12efb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6fa12efb 2024-11-11T07:25:17,738 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T07:25:17,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T07:25:17,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T07:25:17,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T07:25:18,050 INFO [M:0;f334e8b0fbdf:39865 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T07:25:18,051 INFO [M:0;f334e8b0fbdf:39865 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T07:25:18,051 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T07:25:18,051 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:25:18,051 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:25:18,051 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T07:25:18,051 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:25:18,051 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=780.38 KB heapSize=961.38 KB 2024-11-11T07:25:18,066 DEBUG [M:0;f334e8b0fbdf:39865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a16d45bb946e4048b4f3d71f5b4ef671 is 82, key is hbase:meta,,1/info:regioninfo/1731309732214/Put/seqid=0 2024-11-11T07:25:18,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742473_1649 (size=5672) 2024-11-11T07:25:18,470 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2239 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a16d45bb946e4048b4f3d71f5b4ef671 2024-11-11T07:25:18,491 DEBUG [M:0;f334e8b0fbdf:39865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3038076b05924128952a25d6476542f6 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731309882489/Put/seqid=0 2024-11-11T07:25:18,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742474_1650 (size=44543) 2024-11-11T07:25:18,895 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=779.83 KB at sequenceid=2239 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3038076b05924128952a25d6476542f6 2024-11-11T07:25:18,898 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3038076b05924128952a25d6476542f6 2024-11-11T07:25:18,913 DEBUG [M:0;f334e8b0fbdf:39865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0bd72d65ad7c448d8cd50802b4a85954 is 69, key is f334e8b0fbdf,39329,1731309728191/rs:state/1731309730900/Put/seqid=0 2024-11-11T07:25:18,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073742475_1651 (size=5156) 2024-11-11T07:25:19,316 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2239 (bloomFilter=true), to=hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0bd72d65ad7c448d8cd50802b4a85954 2024-11-11T07:25:19,320 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a16d45bb946e4048b4f3d71f5b4ef671 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a16d45bb946e4048b4f3d71f5b4ef671 2024-11-11T07:25:19,322 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a16d45bb946e4048b4f3d71f5b4ef671, entries=8, sequenceid=2239, filesize=5.5 K 2024-11-11T07:25:19,323 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3038076b05924128952a25d6476542f6 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3038076b05924128952a25d6476542f6 2024-11-11T07:25:19,325 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3038076b05924128952a25d6476542f6 2024-11-11T07:25:19,325 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3038076b05924128952a25d6476542f6, entries=172, sequenceid=2239, filesize=43.5 K 2024-11-11T07:25:19,326 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0bd72d65ad7c448d8cd50802b4a85954 as hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0bd72d65ad7c448d8cd50802b4a85954 2024-11-11T07:25:19,328 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40463/user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0bd72d65ad7c448d8cd50802b4a85954, entries=1, sequenceid=2239, filesize=5.0 K 2024-11-11T07:25:19,329 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(3140): Finished flush of dataSize ~780.38 KB/799114, heapSize ~961.09 KB/984152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2239, compaction requested=false 2024-11-11T07:25:19,330 INFO [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:25:19,331 DEBUG [M:0;f334e8b0fbdf:39865 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731309918051Disabling compacts and flushes for region at 1731309918051Disabling writes for close at 1731309918051Obtaining lock to block concurrent updates at 1731309918051Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731309918051Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=799114, getHeapSize=984392, getOffHeapSize=0, getCellsCount=2436 at 1731309918051Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731309918052 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731309918052Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731309918066 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731309918066Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731309918473 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731309918490 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731309918490Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731309918898 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731309918912 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731309918912Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@635efd6a: reopening flushed file at 1731309919319 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@204aa643: reopening flushed file at 1731309919322 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bbe8ca7: reopening flushed file at 1731309919326 (+4 ms)Finished flush of dataSize ~780.38 KB/799114, heapSize ~961.09 KB/984152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2239, compaction requested=false at 1731309919329 (+3 ms)Writing region close event to WAL at 1731309919330 (+1 ms)Closed at 1731309919330 2024-11-11T07:25:19,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40127 is added to blk_1073741830_1006 (size=945856) 2024-11-11T07:25:19,333 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2b6837d3-fd56-2a4c-0184-0f01ebd03537/MasterData/WALs/f334e8b0fbdf,39865,1731309727465/f334e8b0fbdf%2C39865%2C1731309727465.1731309730280 not finished, retry = 0 2024-11-11T07:25:19,434 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T07:25:19,434 INFO [M:0;f334e8b0fbdf:39865 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T07:25:19,434 INFO [M:0;f334e8b0fbdf:39865 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39865 2024-11-11T07:25:19,435 INFO [M:0;f334e8b0fbdf:39865 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T07:25:19,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:25:19,538 INFO [M:0;f334e8b0fbdf:39865 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T07:25:19,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39865-0x10105e7d61c0000, quorum=127.0.0.1:53879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:25:19,544 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T07:25:19,546 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T07:25:19,546 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T07:25:19,546 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T07:25:19,546 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/hadoop.log.dir/,STOPPED} 2024-11-11T07:25:19,549 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T07:25:19,549 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T07:25:19,549 WARN [BP-743239414-172.17.0.2-1731309724451 heartbeating to localhost/127.0.0.1:40463 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T07:25:19,549 WARN [BP-743239414-172.17.0.2-1731309724451 heartbeating to localhost/127.0.0.1:40463 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-743239414-172.17.0.2-1731309724451 (Datanode Uuid 0ee03f57-8736-47de-a7ba-6f3841aa7eb6) service to localhost/127.0.0.1:40463 2024-11-11T07:25:19,552 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143/data/data1/current/BP-743239414-172.17.0.2-1731309724451 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T07:25:19,552 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/cluster_365e91de-f4ed-f3be-8521-2f00144f3143/data/data2/current/BP-743239414-172.17.0.2-1731309724451 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T07:25:19,553 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T07:25:19,560 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T07:25:19,561 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T07:25:19,561 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T07:25:19,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T07:25:19,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/79e8a2aa-0716-e8d7-3f4d-08cbfaf03d77/hadoop.log.dir/,STOPPED} 2024-11-11T07:25:19,577 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T07:25:19,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down